note 110828 modified in function.json-decode by danbrown

If you want to json_decode a large data structure you will run into problems with memory consumption, because ext/json does not currently support "progressive stream reading" like some of the other non-php libs out there. The only thing I found for php that seems to try to do this: https://github.com/shevron/ext-jsonreader but it is not stable and seems no longer developed. So rather than try to implement a messy parser, I found a simplifying workaround. We make some some simple assumptions about the structure of our huge json blob, eg that it is an array of objects at the top level, ie "[{"key":"value",....},{"key":"value2",...}...{"key":"valueN",..}]" The objects can be nested as deep as you like. We can now extract each object from the stream, without reading the whole stream. Then json_decode each one, do sth useful with it and then throw it away. The tricky part is knowing when the object is finished. The normal json_decode can do that for us. We just read to the next "}" and try to decode. If the object is valid that's it, otherwise it is a nested object so read some more..etc. The code below shows how to create a very large json blob in a file (by encoding one object at the time!) and then reading that file back as a stream, one object at the time. The memory consumption of this code is tested to BE CONSTANT. Now matter how many objects you create and read back. The file could be a network socket or STDIN of course..any stream. <?php function formatMem($bytes) { return round($bytes / 1024 / 1024, 2) . 'MB'; } function peakMem() { return formatMem(memory_get_peak_usage()); } echo peakMem() . "\n"; $customer = array( 'title' => 'Mr', 'first_name' => 'John', 'surname' => 'Smith', 'email' => 'john.smith@example.com', 'addresses' => array( array( 'first_line' => '45 Belle Aire Avenue', 'postcode' => 'UT1 OP3', ), ), 'orders' => array( array( 'date' => '2012-12-11', 'currency' => 'GBP', 'total' => 102.56, 'departments' => array( 567, 789, ), ), array( 'date' => '2011-12-11', 'currency' => 'GBP', 'total' => 53.67, 'departments' => array( 123, 456, ), ), ), ); $stream = fopen('/tmp/json', 'w'); fwrite($stream, '['); $quant = 100000; for ($i = 1; $i <= $quant; $i++) { $customer['id'] = $i; fwrite($stream, json_encode($customer)); if ($i < $quant) { fwrite($stream, ','); } } fwrite($stream, ']'); fclose($stream); echo peakMem() . "\n"; // re-read the stream $stream = fopen('/tmp/json', 'r'); $head = stream_get_line($stream, 2); if ($head != '[{') { throw new Exception('not streamable'); } // we think it will be a large array of objects $json_piece = '{'; while (($json_piece .= stream_get_line($stream, 32768, '}')) !== false) { $json_piece .= '}'; // stream_get_line does not return the delim // this is the key step, try to decode, if it is a valid object then we know we are at next master array index $object = json_decode($json_piece); if ($object !== null) { // we sucessfully decoded one object in the master array // now do sth meaningful with $object; echo $object->id . ','; $next_char = stream_get_line($stream, 1); if ($next_char === ']') { // should be at end of file if (($tail = stream_get_line($stream, 32768)) === false || strlen($tail) > 0 || !feof($stream)) { throw new Exception('end of master array but not end of stream, found this: "' . $tail . '"'); } // we are done echo "\n"; break; } elseif($next_char === ',') { // next object in master array..continue $json_piece = ''; } else { throw new Exception('unexpected next char after a complete object "' . $next_char . '"'); } } } fclose($stream); echo peakMem() . "\n"; ?> --was-- If you want to json_decode a large data structure you will run into problems with memory consumption, because ext/json does not currently support "progressive stream reading" like some of the other non-php libs out there. The only thing I found for php that seems to try to do this: https://github.com/shevron/ext-jsonreader but it is not stable and seems no longer developed. So rather than try to implement a messy parser, I found a simplifying workaround. We make some some simple assumptions about the structure of our huge json blob, eg that it is an array of objects at the top level, ie "[{"key":"value",....},{"key":"value2",...}...{"key":"valueN",..}]" The objects can be nested as deep as you like. We can now extract each object from the stream, without reading the whole stream. Then json_decode each one, do sth useful with it and then throw it away. The tricky part is knowing when the object is finished. The normal json_decode can do that for us. We just read to the next "}" and try to decode. If the object is valid that's it, otherwise it is a nested object so read some more..etc. The code below shows how to create a very large json blob in a file (by encoding one object at the time!) and then reading that file back as a stream, one object at the time. The memory consumption of this code is tested to BE CONSTANT. Now matter how many objects you create and read back. The file could be a network socket or STDIN of course..any stream. <?php function formatMem($bytes) { return round($bytes / 1024 / 1024, 2) . 'MB'; } function peakMem() { return formatMem(memory_get_peak_usage()); } echo peakMem() . "\n"; $customer = array( 'title' => 'Mr', 'first_name' => 'John', 'surname' => 'Smith', 'email' => 'john.smith@example.com', 'addresses' => array( array( 'first_line' => '45 Belle Aire Avenue', 'postcode' => 'UT1 OP3', ), ), 'orders' => array( array( 'date' => '2012-12-11', 'currency' => 'GBP', 'total' => 102.56, 'departments' => array( 567, 789, ), ), array( 'date' => '2011-12-11', 'currency' => 'GBP', 'total' => 53.67, 'departments' => array( 123, 456, ), ), ), ); $stream = fopen('/tmp/json', 'w'); fwrite($stream, '['); $quant = 100000; for ($i = 1; $i <= $quant; $i++) { $customer['id'] = $i; fwrite($stream, json_encode($customer)); if ($i < $quant) { fwrite($stream, ','); } } fwrite($stream, ']'); fclose($stream); echo peakMem() . "\n"; // re-read the stream $stream = fopen('/tmp/json', 'r'); $head = stream_get_line($stream, 2); if ($head != '[{') { throw new Exception('not streamable'); } // we think it will be a large array of objects $json_piece = '{'; while (($json_piece .= stream_get_line($stream, 32768, '}')) !== false) { $json_piece .= '}'; // stream_get_line does not return the delim // this is the key step, try to decode, if it is a valid object then we know we are at next master array index $object = json_decode($json_piece); if ($object !== null) { // we sucessfully decoded one object in the master array // now do sth meaningful with $object; echo $object->id . ','; $next_char = stream_get_line($stream, 1); if ($next_char === ']') { // should be at end of file if (($tail = stream_get_line($stream, 32768)) === false || strlen($tail) > 0 || !feof($stream)) { throw new Exception('end of master array but not end of stream, found this: "' . $tail . '"'); } // we are done echo "\n"; break; } elseif($next_char === ',') { // next object in master array..continue $json_piece = ''; } else { throw new Exception('unexpected next char after a complete object "' . $next_char . '"'); } } } fclose($stream); echo peakMem() . "\n"; http://php.net/manual/en/function.json-decode.php