* BREAKING API CHANGE: allow client to specify memory allocation routines

 * bump yajl version to 1.0.0
 * update TODO list, removing completed tasks
 * update yajl_test to override allocation routines and count allocations/frees, providing an automated standalone way to validate we're not leaky
 * update all copyright dates
 * update all .gold files with new test output
 * update ChangeLog in preparation for 1.0.0 release

Author: lloyd

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright 2007, Lloyd Hilaiel.
+# Copyright 2007-2009, Lloyd Hilaiel.
 # 
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -32,9 +32,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6)
 
 PROJECT(YetAnotherJSONParser)
 
-SET (YAJL_MAJOR 0)
-SET (YAJL_MINOR 4)
-SET (YAJL_MICRO 1)
+SET (YAJL_MAJOR 1)
+SET (YAJL_MINOR 0)
+SET (YAJL_MICRO 0)
 
 SET (YAJL_DIST_NAME "yajl-${YAJL_MAJOR}.${YAJL_MINOR}.${YAJL_MICRO}")
 

COPYING

@@ -1,4 +1,4 @@
-Copyright 2007, Lloyd Hilaiel.
+Copyright 2007-2009, Lloyd Hilaiel.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are

ChangeLog

@@ -1,6 +1,17 @@
 1.0.0
      * lth add 'make install' rules, thaks to Andrei Soroker for the
 	contribution.
+     * lth client may override allocation routines at generator or parser
+	allocation time
+     * tjw add yajl_parse_complete routine to allow client to explicitly
+	specify end-of-input, solving the "lonely number" case, where
+	json text consists only of an element with no explicit syntactic
+	end.
+     * tjw many new test cases	
+     * tjw cleanup of code for symmetry and ease of reading
+     * lth integration of patches from Robert Varga which cleanup
+	compilation warnings on 64 bit linux
+	
 0.4.0
      * lth buffer overflow bug in yajl_gen_double s/%lf/%g/ - thanks to
 	Eric Bergstrome

TODO

@@ -1,13 +1,9 @@
 * add a test for 0x1F bug
 * numeric overflow in integers and double
 * line and char offsets in the lexer and in error messages
-* numbers at eof bug. client must have a way of indicating 'end of input'
-  (other than passing in a ' ' whitespace byte),
-  problem illustrated with lonely number test case.
 * testing:
   a. the permuter
   b. some performance comparison against json_checker.
 * investigate pull instead of push parsing
 * Handle memory allocation failures gracefully
-* allow client to override malloc/calloc/free impl.
 * cygwin/msys support on win32

configure

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-# Copyright 2007, Lloyd Hilaiel.
+# Copyright 2007-2009, Lloyd Hilaiel.
 # 
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are

reformatter/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright 2007, Lloyd Hilaiel.
+# Copyright 2007-2009, Lloyd Hilaiel.
 # 
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are

reformatter/json_reformat.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007, Lloyd Hilaiel.
+ * Copyright 2007-2009, Lloyd Hilaiel.
  * 
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -156,10 +156,10 @@ main(int argc, char ** argv)
         usage(argv[0]);
     }
 
-    g = yajl_gen_alloc(&conf);
+    g = yajl_gen_alloc(&conf, NULL);
 
     /* ok.  open file.  let's read and parse */
-    hand = yajl_alloc(&callbacks, &cfg, (void *) g);
+    hand = yajl_alloc(&callbacks, &cfg, (void *) g, NULL);
 
 	while (!done) {
         rd = fread((void *) fileData, 1, sizeof(fileData) - 1, stdin);
@@ -185,7 +185,7 @@ main(int argc, char ** argv)
         {
             unsigned char * str = yajl_get_error(hand, 1, fileData, rd);
             fprintf(stderr, (const char *) str);
-            yajl_free_error(str);
+            yajl_free_error(hand, str);
         } else {
             const unsigned char * buf;
             unsigned int len;

src/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright 2007, Lloyd Hilaiel.
+# Copyright 2007-2009, Lloyd Hilaiel.
 # 
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -28,8 +28,9 @@
 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-SET (SRCS yajl.c yajl_lex.c yajl_parser.c yajl_buf.c yajl_encode.c yajl_gen.c)
-SET (HDRS yajl_parser.h yajl_lex.h yajl_buf.h yajl_encode.h)
+SET (SRCS yajl.c yajl_lex.c yajl_parser.c yajl_buf.c
+          yajl_encode.c yajl_gen.c yajl_alloc.c)
+SET (HDRS yajl_parser.h yajl_lex.h yajl_buf.h yajl_encode.h yajl_alloc.h)
 SET (PUB_HDRS api/yajl_parse.h api/yajl_gen.h api/yajl_common.h)
 
 # useful when fixing lexer bugs.

src/api/yajl_common.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007, Lloyd Hilaiel.
+ * Copyright 2007-2009, Lloyd Hilaiel.
  * 
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -33,6 +33,10 @@
 #ifndef __YAJL_COMMON_H__
 #define __YAJL_COMMON_H__
 
+#ifdef __cplusplus
+extern "C" {
+#endif    
+
 #define YAJL_MAX_DEPTH 128
 
 /* msft dll export gunk.  To build a DLL on windows, you
@@ -48,4 +52,34 @@
 #  define YAJL_API
 #endif 
 
+/** pointer to a malloc function, supporting client overriding memory
+ *  allocation routines */
+typedef void * (*yajl_malloc_func)(void *ctx, unsigned int sz);
+
+/** pointer to a free function, supporting client overriding memory
+ *  allocation routines */
+typedef void (*yajl_free_func)(void *ctx, void * ptr);
+
+/** pointer to a realloc function which can resize an allocation. */
+typedef void * (*yajl_realloc_func)(void *ctx, void * ptr, unsigned int sz);
+
+/** A structure which can be passed to yajl_*_alloc routines to allow the
+ *  client to specify memory allocation functions to be used. */
+typedef struct
+{
+    /** pointer to a function that can allocate uninitialized memory */
+    yajl_malloc_func malloc;
+    /** pointer to a function that can resize memory allocations */
+    yajl_realloc_func realloc;
+    /** pointer to a function that can free memory allocated using
+     *  reallocFunction or mallocFunction */
+    yajl_free_func free;
+    /** a context pointer that will be passed to above allocation routines */
+    void * ctx;
+} yajl_alloc_funcs;
+
+#ifdef __cplusplus
+};
+#endif    
+
 #endif

src/api/yajl_gen.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007, Lloyd Hilaiel.
+ * Copyright 2007-2009, Lloyd Hilaiel.
  * 
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -73,8 +73,18 @@ extern "C" {
         const char * indentString;
     } yajl_gen_config;
 
-    /** allocate a generator handle */
-    yajl_gen YAJL_API yajl_gen_alloc(const yajl_gen_config * config);
+    /** allocate a generator handle
+     *  \param config a pointer to a structure containing parameters which
+     *                configure the behavior of the json generator
+     *  \param allocFuncs an optional pointer to a structure which allows
+     *                    the client to overide the memory allocation
+     *                    used by yajl.  May be NULL, in which case
+     *                    malloc/free/realloc will be used.
+     *
+     *  \returns an allocated handle on success, NULL on failure (bad params)
+     */
+    yajl_gen YAJL_API yajl_gen_alloc(const yajl_gen_config * config,
+                                     const yajl_alloc_funcs * allocFuncs);
 
     /** free a generator handle */    
     void YAJL_API yajl_gen_free(yajl_gen handle);

src/api/yajl_parse.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007, Lloyd Hilaiel.
+ * Copyright 2007-2009, Lloyd Hilaiel.
  * 
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -129,6 +129,7 @@ extern "C" {
      */
     yajl_handle YAJL_API yajl_alloc(const yajl_callbacks * callbacks,
                                     const yajl_parser_config * config,
+                                    const yajl_alloc_funcs * allocFuncs,
                                     void * ctx);
 
     /** free a parser handle */    
@@ -169,7 +170,7 @@ extern "C" {
                                             unsigned int jsonTextLength);
 
     /** free an error returned from yajl_get_error */
-    void YAJL_API yajl_free_error(unsigned char * str);
+    void YAJL_API yajl_free_error(yajl_handle hand, unsigned char * str);
 
 #ifdef __cplusplus
 };

src/yajl

@@ -1,7 +1,7 @@
 /*!
     \mainpage Yet Another JSON Library (YAJL)       
     \author Lloyd Hilaiel
-    \date 2007-2008
+    \date 2007-2009
 
 Yet Another JSON Library (YAJL) is a small event-driven (SAX-style)
 JSON parser written in ANSI C, and a small validating JSON

src/yajl.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007, Lloyd Hilaiel.
+ * Copyright 2007-2009, Lloyd Hilaiel.
  * 
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -33,6 +33,7 @@
 #include "api/yajl_parse.h"
 #include "yajl_lex.h"
 #include "yajl_parser.h"
+#include "yajl_alloc.h"
 
 #include <stdlib.h>
 #include <string.h>
@@ -62,11 +63,29 @@ yajl_status_to_string(yajl_status stat)
 yajl_handle
 yajl_alloc(const yajl_callbacks * callbacks,
            const yajl_parser_config * config,
+           const yajl_alloc_funcs * afs,
            void * ctx)
 {
     unsigned int allowComments = 0;
     unsigned int validateUTF8 = 0;
-    yajl_handle hand = (yajl_handle) malloc(sizeof(struct yajl_handle_t));
+    yajl_handle hand = NULL;
+    yajl_alloc_funcs afsBuffer;
+    
+    /* first order of business is to set up memory allocation routines */
+    if (afs != NULL) {
+        if (afs->malloc == NULL || afs->realloc == NULL || afs->free == NULL)
+        {
+            return NULL;
+        }
+    } else {
+        yajl_set_default_alloc_funcs(&afsBuffer);
+        afs = &afsBuffer;
+    }
+
+    hand = (yajl_handle) YA_MALLOC(afs, sizeof(struct yajl_handle_t));
+
+    /* copy in pointers to allocation routines */
+    memcpy((void *) &(hand->alloc), (void *) afs, sizeof(yajl_alloc_funcs));
 
     if (config != NULL) {
         allowComments = config->allowComments;
@@ -75,10 +94,10 @@ yajl_alloc(const yajl_callbacks * callbacks,
 
     hand->callbacks = callbacks;
     hand->ctx = ctx;
-    hand->lexer = yajl_lex_alloc(allowComments, validateUTF8);
+    hand->lexer = yajl_lex_alloc(&(hand->alloc), allowComments, validateUTF8);
     hand->errorOffset = 0;
-    hand->decodeBuf = yajl_buf_alloc();
-    hand->stateBuf = yajl_buf_alloc();
+    hand->decodeBuf = yajl_buf_alloc(&(hand->alloc));
+    hand->stateBuf = yajl_buf_alloc(&(hand->alloc));
 
     yajl_state_push(hand, yajl_state_start);    
 
@@ -91,7 +110,7 @@ yajl_free(yajl_handle handle)
     yajl_buf_free(handle->stateBuf);
     yajl_buf_free(handle->decodeBuf);
     yajl_lex_free(handle->lexer);
-    free(handle);
+    YA_FREE(&(handle->alloc), handle);
 }
 
 yajl_status
@@ -124,10 +143,10 @@ yajl_get_error(yajl_handle hand, int verbose,
 }
 
 void
-yajl_free_error(unsigned char * str)
+yajl_free_error(yajl_handle hand, unsigned char * str)
 {
-    /* XXX: use memory allocation functions if set */
-    free(str);
+    /* use memory allocation functions if set */
+    YA_FREE(&(hand->alloc), str);
 }
 
 /* XXX: add utility routines to parse from file */

src/yajl_alloc.c

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2007-2009, Lloyd Hilaiel.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ * 
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ * 
+ *  3. Neither the name of Lloyd Hilaiel nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */ 
+
+/**
+ * \file yajl_alloc.h
+ * default memory allocation routines for yajl which use malloc/realloc and
+ * free
+ */
+
+#include "yajl_alloc.h"
+#include <stdlib.h>
+
+static void * yajl_internal_malloc(void *ctx, unsigned int sz)
+{
+    return malloc(sz);
+}
+
+static void * yajl_internal_realloc(void *ctx, void * previous,
+                                    unsigned int sz)
+{
+    return realloc(previous, sz);
+}
+
+static void yajl_internal_free(void *ctx, void * ptr)
+{
+    free(ptr);
+}
+
+void yajl_set_default_alloc_funcs(yajl_alloc_funcs * yaf)
+{
+    yaf->malloc = yajl_internal_malloc;
+    yaf->free = yajl_internal_free;
+    yaf->realloc = yajl_internal_realloc;
+    yaf->ctx = NULL;
+}
+