/src/glaze/include/glaze/json/ndjson.hpp

Source
// Glaze Library
// For the license information refer to glaze.hpp

#pragma once

#include <cstring>

#include "glaze/json/read.hpp"
#include "glaze/json/write.hpp"

namespace glz
{
   template <>
   struct parse<NDJSON>
   {
      template <auto Opts, class T, is_context Ctx, class It0, class It1>
      static void op(T&& value, Ctx&& ctx, It0&& it, It1&& end)
      {
         from<NDJSON, std::remove_reference_t<T>>::template op<Opts>(std::forward<T>(value), std::forward<Ctx>(ctx),
                                                                     std::forward<It0>(it), std::forward<It1>(end));
      }
   };

   template <>
   struct serialize<NDJSON>
   {
      template <auto Opts, class T, is_context Ctx, class B, class IX>
      static void op(T&& value, Ctx&& ctx, B&& b, IX&& ix)
      {
         to<NDJSON, std::decay_t<T>>::template op<Opts>(std::forward<T>(value), std::forward<Ctx>(ctx),
                                                        std::forward<B>(b), std::forward<IX>(ix));
      }
   };

   // Consume the line breaks that separate two records.
   //
   // Both line endings are handled by one loop rather than one after the other, so that a CRLF
   // following a bare LF is still a separator. Scanning for all the CRs and then all the LFs stops
   // at the first CR after an LF and calls it the start of a record.
   //
   // A '\r' whose '\n' has not arrived is left unconsumed rather than rejected. From in here a CRLF
   // split by the window edge and a stray carriage return are the same two bytes; only the caller
   // knows whether more input can still arrive. So this stops there without an error, and the
   // caller either refills and calls again or reports the syntax error itself.
   //
   // A null-terminated buffer stops on the trailing '\0' sentinel. A non-null-terminated buffer has
   // no sentinel, so bound the scan on it != end.
   template <auto Opts>
   GLZ_ALWAYS_INLINE void skip_record_separators(is_context auto& ctx, auto& it, auto& end) noexcept
   {
      if constexpr (Opts.null_terminated) {
         while (true) {
            if (*it == '\n') {
               ++it;
               continue;
            }
            if (*it == '\r') {
               if (*(it + 1) == '\n') {
                  it += 2;
                  continue;
               }
               ctx.error = error_code::syntax_error; // Expected '\n' after '\r'
            }
            return;
         }
      }
      else {
         while (it != end) {
            if (*it == '\n') {
               ++it;
               continue;
            }
            if (*it == '\r') {
               if ((it + 1) == end) {
                  return; // undecidable from this window; the caller refills or errors
               }
               if (*(it + 1) == '\n') {
                  it += 2;
                  continue;
               }
               ctx.error = error_code::syntax_error; // Expected '\n' after '\r'
            }
            return;
         }
      }
   }

   // Whether a record delimiter is anywhere in the window ahead of `it`.
   //
   // Either line ending closes a record, and '\r' has to count on its own: a CRLF can be split by
   // the window edge, and a window holding the whole record plus the '\r' does hold a complete
   // record. Requiring the '\n' would call that record unreadable while it is sitting right there.
   //
   // A raw '\r' or '\n' inside a JSON string is invalid -- both have to be escaped -- so the first
   // one after the start of a record always ends it.
   GLZ_ALWAYS_INLINE bool window_holds_record_end(const char* it, const char* end) noexcept
   {
      const size_t n = size_t(end - it);
      if (n == 0) {
         return false;
      }
      return std::memchr(it, '\n', n) != nullptr || std::memchr(it, '\r', n) != nullptr;
   }

   // Bring a whole record into the window, so the record reader is never handed a partial one.
   //
   // Records are newline delimited, which makes "does the window hold a complete record" a question
   // that can be answered before parsing: look for the delimiter. Nothing weaker works. A refill
   // policy driven by a byte count cannot tell "the window ran out in the middle of this record"
   // from "this record ended with the buffer", and the JSON reader reports both the same way, as a
   // value that parsed cleanly up to `end`. A bare number split by the window edge then reads as
   // two numbers, and neither the reader nor the caller has anything to notice it by.
   //
   // Refilling until the delimiter appears also drops the ceiling a byte count imposes. A record no
   // longer has to fit in whatever fraction of the window happened to be left over from the last
   // one; it only has to fit in the window.
   //
   // Returns false when no refill can produce a delimiter, which means the record really is wider
   // than the window -- the standing limit of streaming, since nothing refills inside a record.
   template <class Ctx>
   bool fill_window_to_record_end(Ctx& ctx, auto& it, auto& end) noexcept
   {
      if constexpr (has_streaming_state<Ctx>) {
         if (ctx.stream.enabled()) {
            while (!window_holds_record_end(it, end)) {
               // The last record in a document may end without a delimiter, so a source with
               // nothing left to give has already delivered the whole record.
               if (ctx.stream.source_at_eof()) {
                  return true;
               }
               const size_t available = size_t(end - it);
               refill_window(ctx, it, end);
               if (size_t(end - it) <= available) {
                  return false; // the refill brought nothing new: the record does not fit
               }
            }
         }
      }
      return true;
   }

   // Position `it` at the start of the next record, with the whole record in the window, and report
   // whether there is one.
   //
   // Records are independent, so the gap between two of them is the only place a streaming window
   // may release what has been parsed and pull in more. Doing that here is what lets a document
   // larger than the window be read at all; without it the read stops at the first window's edge,
   // carrying only the records that happened to fit and calling that a complete document.
   //
   // Returns false when the input is exhausted, when the separators were malformed, and when a
   // record is wider than the window; the caller tells them apart from ctx.error, which is what
   // distinguishes a document that ended from one that could not be read.
   template <auto Opts, class Ctx>
   bool ndjson_has_next_record(Ctx& ctx, auto& it, auto& end) noexcept
   {
      // A run of separators can be wider than the window, so this loops: each pass either reaches a
      // record byte or releases the separators it walked and pulls more input in behind them.
      while (true) {
         skip_record_separators<Opts>(ctx, it, end);
         if (bool(ctx.error)) [[unlikely]] {
            return false;
         }
         if (it < end && *it != '\r') {
            break; // a record byte
         }

         // Either the window is spent, or it ends on a '\r' whose '\n' has not arrived. Both are
         // answered by more input, and both are the end of the document if there is none.
         bool progressed = false;
         if constexpr (has_streaming_state<Ctx>) {
            if (ctx.stream.enabled() && !ctx.stream.source_at_eof()) {
               const size_t available = size_t(end - it);
               refill_window(ctx, it, end);
               progressed = size_t(end - it) > available;
            }
         }
         if (!progressed) {
            if (it < end) [[unlikely]] {
               // a '\r' at the true end of the input, with no '\n' to pair it with
               ctx.error = error_code::syntax_error;
            }
            return false;
         }
      }

      if (!fill_window_to_record_end(ctx, it, end)) {
         ctx.error = error_code::streaming_unsupported;
         ctx.custom_error_message =
            "this NDJSON record is wider than the buffer window; a record must fit in the window "
            "because nothing refills inside one";
         return false;
      }
      return true;
   }

   // Read one record into `record`. Returns false on error.
   //
   // A record that closed cleanly leaves ctx.depth at zero, so end_reached there means the input
   // ran out exactly where the record ended rather than in the middle of it -- the record is
   // complete either way, and whether anything follows it is the next iteration's question.
   //
   // Any other way of running out of input means the record itself was cut short. For a buffer that
   // is a truncated document. For a stream with input still pending it is instead the window that
   // ran out: the JSON reader refills between the members of an object and the elements of an
   // array, but nothing refills in the middle of a string or a number, so a single token has to fit
   // in the window. Naming that beats reporting a truncated document, since the document is fine
   // and the buffer is the thing to change.
   template <auto Opts, class Ctx>
   bool read_ndjson_record(auto&& record, Ctx& ctx, auto& it, auto& end)
   {
      parse<JSON>::op<Opts>(record, ctx, it, end);
      resync_window_end(ctx, end); // the JSON reader may have refilled inside the record

      if (!bool(ctx.error)) [[likely]] {
         return true;
      }

      // "The input ran out exactly where this record ended" is only the same statement as "this
      // record is complete" when the input that ran out was the document. If it was the window, the
      // record was cut mid-token and the bytes after the cut are the rest of that same token --
      // accepting it here splits one value into two and reports success, which is the one outcome a
      // reader must never produce. ndjson_has_next_record is what normally prevents this, by
      // refusing to start a record it cannot see the end of; this is the check that makes the
      // guarantee local rather than something the caller has to be trusted to have arranged.
      if (ctx.error == error_code::end_reached && ctx.depth == 0) {
         bool cut_by_window = false;
         if constexpr (has_streaming_state<Ctx>) {
            cut_by_window = ctx.stream.enabled() && !ctx.stream.source_at_eof() && it >= end;
         }
         if (!cut_by_window) {
            ctx.error = error_code::none;
            return true;
         }
      }

      if constexpr (has_streaming_state<Ctx>) {
         const bool ran_out_of_input = ctx.error == error_code::end_reached || ctx.error == error_code::unexpected_end;
         if (ran_out_of_input && ctx.stream.enabled() && !ctx.stream.source_at_eof()) {
            ctx.error = error_code::streaming_unsupported;
            ctx.custom_error_message =
               "this NDJSON record is wider than the buffer window; a record must fit in the window "
               "because nothing refills inside one";
         }
      }
      return false;
   }

   template <class T>
      requires readable_array_t<T> && (emplace_backable<T> || !resizable<T>)
   struct from<NDJSON, T>
   {
      template <auto Opts>
      static void op(auto& value, is_context auto&& ctx, auto&& it, auto end)
      {
         if (bool(ctx.error)) [[unlikely]] {
            return;
         }

         if (it == end) {
            if constexpr (resizable<T>) {
               value.clear();

               if constexpr (check_shrink_to_fit(Opts) && has_shrink_to_fit<T>) {
                  value.shrink_to_fit();
               }
            }
         }

         const auto n = value.size();

         auto value_it = value.begin();

         const auto truncate_to = [&](auto first_unwritten) {
            if constexpr (erasable<T>) {
               // erase rather than resize, for element types that are not default constructible
               value.erase(first_unwritten, value.end());

               if constexpr (check_shrink_to_fit(Opts) && has_shrink_to_fit<T>) {
                  value.shrink_to_fit();
               }
            }
         };

         for (size_t i = 0; i < n; ++i) {
            if (!ndjson_has_next_record<Opts>(ctx, it, end)) {
               if (bool(ctx.error)) [[unlikely]] {
                  return;
               }
               truncate_to(value_it);
               return;
            }
            if (!read_ndjson_record<Opts>(*value_it, ctx, it, end)) [[unlikely]] {
               return;
            }
            ++value_it;
         }

         // growing
         if constexpr (emplace_backable<T>) {
            while (ndjson_has_next_record<Opts>(ctx, it, end)) {
               if (!read_ndjson_record<Opts>(value.emplace_back(), ctx, it, end)) [[unlikely]] {
                  return;
               }
            }
            if (bool(ctx.error)) [[unlikely]] {
               return; // malformed separators, not the end of the document
            }

            if constexpr (check_shrink_to_fit(Opts) && has_shrink_to_fit<T>) {
               value.shrink_to_fit();
            }
         }
         else {
            ctx.error = error_code::exceeded_static_array_size;
         }
      }
   };

   template <class T>
      requires glaze_array_t<T> || tuple_t<T> || is_std_tuple<T>
   struct from<NDJSON, T>
   {
      template <auto Opts>
      static void op(auto& value, is_context auto&& ctx, auto&& it, auto end)
      {
         if (bool(ctx.error)) [[unlikely]] {
            return;
         }

         static constexpr auto N = []() constexpr {
            if constexpr (glaze_array_t<T>) {
               return reflect<T>::size;
            }
            else {
               return glz::tuple_size_v<T>;
            }
         }();

         for_each<N>([&]<auto I>() {
            if (bool(ctx.error) || !ndjson_has_next_record<Opts>(ctx, it, end)) {
               return; // for_each has no early exit; the guard above stands in for one
            }
            if constexpr (is_std_tuple<T>) {
               (void)read_ndjson_record<Opts>(std::get<I>(value), ctx, it, end);
            }
            else if constexpr (glaze_array_t<T>) {
               (void)read_ndjson_record<Opts>(get_member(value, glz::get<I>(meta_v<T>)), ctx, it, end);
            }
            else {
               (void)read_ndjson_record<Opts>(glz::get<I>(value), ctx, it, end);
            }
         });
      }
   };

   template <writable_array_t T>
   struct to<NDJSON, T>
   {
      template <auto Opts, class... Args>
      static void op(auto&& value, is_context auto&& ctx, auto&& b, auto& ix)
      {
         const auto is_empty = [&]() -> bool {
            if constexpr (has_size<T>) {
               return value.size() ? false : true;
            }
            else {
               return value.empty();
            }
         }();

         if (!is_empty) {
            auto it = value.begin();
            using Value = core_t<decltype(*it)>;
            to<JSON, Value>::template op<Opts>(*it, ctx, b, ix);
            ++it;
            const auto end = value.end();
            for (; it != end; ++it) {
               dump('\n', b, ix);
               to<JSON, Value>::template op<Opts>(*it, ctx, b, ix);
            }
         }
      }
   };

   template <class T>
      requires glaze_array_t<T> || tuple_t<T>
   struct to<NDJSON, T>
   {
      template <auto Opts, class... Args>
      static void op(auto&& value, is_context auto&& ctx, Args&&... args)
      {
         static constexpr auto N = []() constexpr {
            if constexpr (glaze_array_t<std::decay_t<T>>) {
               return glz::tuple_size_v<meta_t<std::decay_t<T>>>;
            }
            else {
               return glz::tuple_size_v<std::decay_t<T>>;
            }
         }();

         using V = std::decay_t<T>;
         for_each<N>([&]<auto I>() {
            if constexpr (glaze_array_t<V>) {
               serialize<JSON>::op<Opts>(get_member(value, glz::get<I>(meta_v<T>)), ctx, args...);
            }
            else {
               serialize<JSON>::op<Opts>(glz::get<I>(value), ctx, args...);
            }
            constexpr bool needs_new_line = I < N - 1;
            if constexpr (needs_new_line) {
               dump('\n', args...);
            }
         });
      }
   };

   template <class T>
      requires is_std_tuple<std::decay_t<T>>
   struct to<NDJSON, T>
   {
      template <auto Opts, class... Args>
      static void op(auto&& value, is_context auto&& ctx, Args&&... args)
      {
         static constexpr auto N = []() constexpr {
            if constexpr (glaze_array_t<std::decay_t<T>>) {
               return glz::tuple_size_v<meta_t<std::decay_t<T>>>;
            }
            else {
               return glz::tuple_size_v<std::decay_t<T>>;
            }
         }();

         using V = std::decay_t<T>;
         for_each<N>([&]<auto I>() {
            if constexpr (glaze_array_t<V>) {
               serialize<JSON>::op<Opts>(value.*std::get<I>(meta_v<V>), ctx, std::forward<Args>(args)...);
            }
            else {
               serialize<JSON>::op<Opts>(std::get<I>(value), ctx, std::forward<Args>(args)...);
            }
            constexpr bool needs_new_line = I < N - 1;
            if constexpr (needs_new_line) {
               dump('\n', std::forward<Args>(args)...);
            }
         });
      }
   };

   template <read_supported<NDJSON> T, class Buffer>
   [[nodiscard]] auto read_ndjson(T& value, Buffer&& buffer)
   {
      context ctx{};
      return read<opts{.format = NDJSON}>(value, std::forward<Buffer>(buffer), ctx);
   }

   template <read_supported<NDJSON> T, class Buffer>
   [[nodiscard]] expected<T, error_ctx> read_ndjson(Buffer&& buffer)
   {
      T value{};
      context ctx{};
      const auto ec = read<opts{.format = NDJSON}>(value, std::forward<Buffer>(buffer), ctx);
      if (ec == error_code::none) {
         return value;
      }
      return unexpected(ec);
   }

   template <auto Opts = opts{.format = NDJSON}, read_supported<NDJSON> T>
   [[nodiscard]] error_ctx read_file_ndjson(T& value, const sv file_name)
   {
      context ctx{};
      ctx.current_file = file_name;

      std::string buffer;

      const auto ec = file_to_buffer(buffer, ctx.current_file);

      if (bool(ec)) {
         return {0, ec};
      }

      return read<Opts>(value, buffer, ctx);
   }

   template <write_supported<NDJSON> T, class Buffer>
   [[nodiscard]] error_ctx write_ndjson(T&& value, Buffer&& buffer)
   {
      return write<opts{.format = NDJSON}>(std::forward<T>(value), std::forward<Buffer>(buffer));
   }

   template <write_supported<NDJSON> T>
   [[nodiscard]] expected<std::string, error_ctx> write_ndjson(T&& value)
   {
      return write<opts{.format = NDJSON}>(std::forward<T>(value));
   }

   template <write_supported<NDJSON> T>
   [[nodiscard]] error_ctx write_file_ndjson(T&& value, const std::string& file_name, auto&& buffer)
   {
      const auto ec = write<opts{.format = NDJSON}>(std::forward<T>(value), buffer);
      if (bool(ec)) [[unlikely]] {
         return ec;
      }
      const auto file_ec = buffer_to_file(buffer, file_name);
      if (bool(file_ec)) [[unlikely]] {
         return {0, file_ec};
      }
      return {buffer.size(), error_code::none};
   }
}

Coverage Report

Created: 2026-09-01 06:56

Line	Count	Source
1		// Glaze Library
2		// For the license information refer to glaze.hpp
3
4		#pragma once
5
6		#include <cstring>
7
8		#include "glaze/json/read.hpp"
9		#include "glaze/json/write.hpp"
10
11		namespace glz
12		{
13		template <>
14		struct parse<NDJSON>
15		{
16		template <auto Opts, class T, is_context Ctx, class It0, class It1>
17		static void op(T&& value, Ctx&& ctx, It0&& it, It1&& end)
18		{
19		from<NDJSON, std::remove_reference_t<T>>::template op<Opts>(std::forward<T>(value), std::forward<Ctx>(ctx),
20		std::forward<It0>(it), std::forward<It1>(end));
21		}
22		};
23
24		template <>
25		struct serialize<NDJSON>
26		{
27		template <auto Opts, class T, is_context Ctx, class B, class IX>
28		static void op(T&& value, Ctx&& ctx, B&& b, IX&& ix)
29		{
30		to<NDJSON, std::decay_t<T>>::template op<Opts>(std::forward<T>(value), std::forward<Ctx>(ctx),
31		std::forward<B>(b), std::forward<IX>(ix));
32		}
33		};
34
35		// Consume the line breaks that separate two records.
36		//
37		// Both line endings are handled by one loop rather than one after the other, so that a CRLF
38		// following a bare LF is still a separator. Scanning for all the CRs and then all the LFs stops
39		// at the first CR after an LF and calls it the start of a record.
40		//
41		// A '\r' whose '\n' has not arrived is left unconsumed rather than rejected. From in here a CRLF
42		// split by the window edge and a stray carriage return are the same two bytes; only the caller
43		// knows whether more input can still arrive. So this stops there without an error, and the
44		// caller either refills and calls again or reports the syntax error itself.
45		//
46		// A null-terminated buffer stops on the trailing '\0' sentinel. A non-null-terminated buffer has
47		// no sentinel, so bound the scan on it != end.
48		template <auto Opts>
49		GLZ_ALWAYS_INLINE void skip_record_separators(is_context auto& ctx, auto& it, auto& end) noexcept
50		{
51		if constexpr (Opts.null_terminated) {
52		while (true) {
53		if (*it == '\n') {
54		++it;
55		continue;
56		}
57		if (*it == '\r') {
58		if (*(it + 1) == '\n') {
59		it += 2;
60		continue;
61		}
62		ctx.error = error_code::syntax_error; // Expected '\n' after '\r'
63		}
64		return;
65		}
66		}
67		else {
68		while (it != end) {
69		if (*it == '\n') {
70		++it;
71		continue;
72		}
73		if (*it == '\r') {
74		if ((it + 1) == end) {
75		return; // undecidable from this window; the caller refills or errors
76		}
77		if (*(it + 1) == '\n') {
78		it += 2;
79		continue;
80		}
81		ctx.error = error_code::syntax_error; // Expected '\n' after '\r'
82		}
83		return;
84		}
85		}
86		}
87
88		// Whether a record delimiter is anywhere in the window ahead of `it`.
89		//
90		// Either line ending closes a record, and '\r' has to count on its own: a CRLF can be split by
91		// the window edge, and a window holding the whole record plus the '\r' does hold a complete
92		// record. Requiring the '\n' would call that record unreadable while it is sitting right there.
93		//
94		// A raw '\r' or '\n' inside a JSON string is invalid -- both have to be escaped -- so the first
95		// one after the start of a record always ends it.
96		GLZ_ALWAYS_INLINE bool window_holds_record_end(const char* it, const char* end) noexcept
97	0	{
98	0	const size_t n = size_t(end - it);
99	0	if (n == 0) {
100	0	return false;
101	0	}
102	0	return std::memchr(it, '\n', n) != nullptr \|\| std::memchr(it, '\r', n) != nullptr;
103	0	}
104
105		// Bring a whole record into the window, so the record reader is never handed a partial one.
106		//
107		// Records are newline delimited, which makes "does the window hold a complete record" a question
108		// that can be answered before parsing: look for the delimiter. Nothing weaker works. A refill
109		// policy driven by a byte count cannot tell "the window ran out in the middle of this record"
110		// from "this record ended with the buffer", and the JSON reader reports both the same way, as a
111		// value that parsed cleanly up to `end`. A bare number split by the window edge then reads as
112		// two numbers, and neither the reader nor the caller has anything to notice it by.
113		//
114		// Refilling until the delimiter appears also drops the ceiling a byte count imposes. A record no
115		// longer has to fit in whatever fraction of the window happened to be left over from the last
116		// one; it only has to fit in the window.
117		//
118		// Returns false when no refill can produce a delimiter, which means the record really is wider
119		// than the window -- the standing limit of streaming, since nothing refills inside a record.
120		template <class Ctx>
121		bool fill_window_to_record_end(Ctx& ctx, auto& it, auto& end) noexcept
122		{
123		if constexpr (has_streaming_state<Ctx>) {
124		if (ctx.stream.enabled()) {
125		while (!window_holds_record_end(it, end)) {
126		// The last record in a document may end without a delimiter, so a source with
127		// nothing left to give has already delivered the whole record.
128		if (ctx.stream.source_at_eof()) {
129		return true;
130		}
131		const size_t available = size_t(end - it);
132		refill_window(ctx, it, end);
133		if (size_t(end - it) <= available) {
134		return false; // the refill brought nothing new: the record does not fit
135		}
136		}
137		}
138		}
139		return true;
140		}
141
142		// Position `it` at the start of the next record, with the whole record in the window, and report
143		// whether there is one.
144		//
145		// Records are independent, so the gap between two of them is the only place a streaming window
146		// may release what has been parsed and pull in more. Doing that here is what lets a document
147		// larger than the window be read at all; without it the read stops at the first window's edge,
148		// carrying only the records that happened to fit and calling that a complete document.
149		//
150		// Returns false when the input is exhausted, when the separators were malformed, and when a
151		// record is wider than the window; the caller tells them apart from ctx.error, which is what
152		// distinguishes a document that ended from one that could not be read.
153		template <auto Opts, class Ctx>
154		bool ndjson_has_next_record(Ctx& ctx, auto& it, auto& end) noexcept
155		{
156		// A run of separators can be wider than the window, so this loops: each pass either reaches a
157		// record byte or releases the separators it walked and pulls more input in behind them.
158		while (true) {
159		skip_record_separators<Opts>(ctx, it, end);
160		if (bool(ctx.error)) [[unlikely]] {
161		return false;
162		}
163		if (it < end && *it != '\r') {
164		break; // a record byte
165		}
166
167		// Either the window is spent, or it ends on a '\r' whose '\n' has not arrived. Both are
168		// answered by more input, and both are the end of the document if there is none.
169		bool progressed = false;
170		if constexpr (has_streaming_state<Ctx>) {
171		if (ctx.stream.enabled() && !ctx.stream.source_at_eof()) {
172		const size_t available = size_t(end - it);
173		refill_window(ctx, it, end);
174		progressed = size_t(end - it) > available;
175		}
176		}
177		if (!progressed) {
178		if (it < end) [[unlikely]] {
179		// a '\r' at the true end of the input, with no '\n' to pair it with
180		ctx.error = error_code::syntax_error;
181		}
182		return false;
183		}
184		}
185
186		if (!fill_window_to_record_end(ctx, it, end)) {
187		ctx.error = error_code::streaming_unsupported;
188		ctx.custom_error_message =
189		"this NDJSON record is wider than the buffer window; a record must fit in the window "
190		"because nothing refills inside one";
191		return false;
192		}
193		return true;
194		}
195
196		// Read one record into `record`. Returns false on error.
197		//
198		// A record that closed cleanly leaves ctx.depth at zero, so end_reached there means the input
199		// ran out exactly where the record ended rather than in the middle of it -- the record is
200		// complete either way, and whether anything follows it is the next iteration's question.
201		//
202		// Any other way of running out of input means the record itself was cut short. For a buffer that
203		// is a truncated document. For a stream with input still pending it is instead the window that
204		// ran out: the JSON reader refills between the members of an object and the elements of an
205		// array, but nothing refills in the middle of a string or a number, so a single token has to fit
206		// in the window. Naming that beats reporting a truncated document, since the document is fine
207		// and the buffer is the thing to change.
208		template <auto Opts, class Ctx>
209		bool read_ndjson_record(auto&& record, Ctx& ctx, auto& it, auto& end)
210		{
211		parse<JSON>::op<Opts>(record, ctx, it, end);
212		resync_window_end(ctx, end); // the JSON reader may have refilled inside the record
213
214		if (!bool(ctx.error)) [[likely]] {
215		return true;
216		}
217
218		// "The input ran out exactly where this record ended" is only the same statement as "this
219		// record is complete" when the input that ran out was the document. If it was the window, the
220		// record was cut mid-token and the bytes after the cut are the rest of that same token --
221		// accepting it here splits one value into two and reports success, which is the one outcome a
222		// reader must never produce. ndjson_has_next_record is what normally prevents this, by
223		// refusing to start a record it cannot see the end of; this is the check that makes the
224		// guarantee local rather than something the caller has to be trusted to have arranged.
225		if (ctx.error == error_code::end_reached && ctx.depth == 0) {
226		bool cut_by_window = false;
227		if constexpr (has_streaming_state<Ctx>) {
228		cut_by_window = ctx.stream.enabled() && !ctx.stream.source_at_eof() && it >= end;
229		}
230		if (!cut_by_window) {
231		ctx.error = error_code::none;
232		return true;
233		}
234		}
235
236		if constexpr (has_streaming_state<Ctx>) {
237		const bool ran_out_of_input = ctx.error == error_code::end_reached \|\| ctx.error == error_code::unexpected_end;
238		if (ran_out_of_input && ctx.stream.enabled() && !ctx.stream.source_at_eof()) {
239		ctx.error = error_code::streaming_unsupported;
240		ctx.custom_error_message =
241		"this NDJSON record is wider than the buffer window; a record must fit in the window "
242		"because nothing refills inside one";
243		}
244		}
245		return false;
246		}
247
248		template <class T>
249		requires readable_array_t<T> && (emplace_backable<T> \|\| !resizable<T>)
250		struct from<NDJSON, T>
251		{
252		template <auto Opts>
253		static void op(auto& value, is_context auto&& ctx, auto&& it, auto end)
254		{
255		if (bool(ctx.error)) [[unlikely]] {
256		return;
257		}
258
259		if (it == end) {
260		if constexpr (resizable<T>) {
261		value.clear();
262
263		if constexpr (check_shrink_to_fit(Opts) && has_shrink_to_fit<T>) {
264		value.shrink_to_fit();
265		}
266		}
267		}
268
269		const auto n = value.size();
270
271		auto value_it = value.begin();
272
273		const auto truncate_to = [&](auto first_unwritten) {
274		if constexpr (erasable<T>) {
275		// erase rather than resize, for element types that are not default constructible
276		value.erase(first_unwritten, value.end());
277
278		if constexpr (check_shrink_to_fit(Opts) && has_shrink_to_fit<T>) {
279		value.shrink_to_fit();
280		}
281		}
282		};
283
284		for (size_t i = 0; i < n; ++i) {
285		if (!ndjson_has_next_record<Opts>(ctx, it, end)) {
286		if (bool(ctx.error)) [[unlikely]] {
287		return;
288		}
289		truncate_to(value_it);
290		return;
291		}
292		if (!read_ndjson_record<Opts>(*value_it, ctx, it, end)) [[unlikely]] {
293		return;
294		}
295		++value_it;
296		}
297
298		// growing
299		if constexpr (emplace_backable<T>) {
300		while (ndjson_has_next_record<Opts>(ctx, it, end)) {
301		if (!read_ndjson_record<Opts>(value.emplace_back(), ctx, it, end)) [[unlikely]] {
302		return;
303		}
304		}
305		if (bool(ctx.error)) [[unlikely]] {
306		return; // malformed separators, not the end of the document
307		}
308
309		if constexpr (check_shrink_to_fit(Opts) && has_shrink_to_fit<T>) {
310		value.shrink_to_fit();
311		}
312		}
313		else {
314		ctx.error = error_code::exceeded_static_array_size;
315		}
316		}
317		};
318
319		template <class T>
320		requires glaze_array_t<T> \|\| tuple_t<T> \|\| is_std_tuple<T>
321		struct from<NDJSON, T>
322		{
323		template <auto Opts>
324		static void op(auto& value, is_context auto&& ctx, auto&& it, auto end)
325		{
326		if (bool(ctx.error)) [[unlikely]] {
327		return;
328		}
329
330		static constexpr auto N = []() constexpr {
331		if constexpr (glaze_array_t<T>) {
332		return reflect<T>::size;
333		}
334		else {
335		return glz::tuple_size_v<T>;
336		}
337		}();
338
339		for_each<N>([&]<auto I>() {
340		if (bool(ctx.error) \|\| !ndjson_has_next_record<Opts>(ctx, it, end)) {
341		return; // for_each has no early exit; the guard above stands in for one
342		}
343		if constexpr (is_std_tuple<T>) {
344		(void)read_ndjson_record<Opts>(std::get<I>(value), ctx, it, end);
345		}
346		else if constexpr (glaze_array_t<T>) {
347		(void)read_ndjson_record<Opts>(get_member(value, glz::get<I>(meta_v<T>)), ctx, it, end);
348		}
349		else {
350		(void)read_ndjson_record<Opts>(glz::get<I>(value), ctx, it, end);
351		}
352		});
353		}
354		};
355
356		template <writable_array_t T>
357		struct to<NDJSON, T>
358		{
359		template <auto Opts, class... Args>
360		static void op(auto&& value, is_context auto&& ctx, auto&& b, auto& ix)
361		{
362		const auto is_empty = [&]() -> bool {
363		if constexpr (has_size<T>) {
364		return value.size() ? false : true;
365		}
366		else {
367		return value.empty();
368		}
369		}();
370
371		if (!is_empty) {
372		auto it = value.begin();
373		using Value = core_t<decltype(*it)>;
374		to<JSON, Value>::template op<Opts>(*it, ctx, b, ix);
375		++it;
376		const auto end = value.end();
377		for (; it != end; ++it) {
378		dump('\n', b, ix);
379		to<JSON, Value>::template op<Opts>(*it, ctx, b, ix);
380		}
381		}
382		}
383		};
384
385		template <class T>
386		requires glaze_array_t<T> \|\| tuple_t<T>
387		struct to<NDJSON, T>
388		{
389		template <auto Opts, class... Args>
390		static void op(auto&& value, is_context auto&& ctx, Args&&... args)
391		{
392		static constexpr auto N = []() constexpr {
393		if constexpr (glaze_array_t<std::decay_t<T>>) {
394		return glz::tuple_size_v<meta_t<std::decay_t<T>>>;
395		}
396		else {
397		return glz::tuple_size_v<std::decay_t<T>>;
398		}
399		}();
400
401		using V = std::decay_t<T>;
402		for_each<N>([&]<auto I>() {
403		if constexpr (glaze_array_t<V>) {
404		serialize<JSON>::op<Opts>(get_member(value, glz::get<I>(meta_v<T>)), ctx, args...);
405		}
406		else {
407		serialize<JSON>::op<Opts>(glz::get<I>(value), ctx, args...);
408		}
409		constexpr bool needs_new_line = I < N - 1;
410		if constexpr (needs_new_line) {
411		dump('\n', args...);
412		}
413		});
414		}
415		};
416
417		template <class T>
418		requires is_std_tuple<std::decay_t<T>>
419		struct to<NDJSON, T>
420		{
421		template <auto Opts, class... Args>
422		static void op(auto&& value, is_context auto&& ctx, Args&&... args)
423		{
424		static constexpr auto N = []() constexpr {
425		if constexpr (glaze_array_t<std::decay_t<T>>) {
426		return glz::tuple_size_v<meta_t<std::decay_t<T>>>;
427		}
428		else {
429		return glz::tuple_size_v<std::decay_t<T>>;
430		}
431		}();
432
433		using V = std::decay_t<T>;
434		for_each<N>([&]<auto I>() {
435		if constexpr (glaze_array_t<V>) {
436		serialize<JSON>::op<Opts>(value.*std::get<I>(meta_v<V>), ctx, std::forward<Args>(args)...);
437		}
438		else {
439		serialize<JSON>::op<Opts>(std::get<I>(value), ctx, std::forward<Args>(args)...);
440		}
441		constexpr bool needs_new_line = I < N - 1;
442		if constexpr (needs_new_line) {
443		dump('\n', std::forward<Args>(args)...);
444		}
445		});
446		}
447		};
448
449		template <read_supported<NDJSON> T, class Buffer>
450		[[nodiscard]] auto read_ndjson(T& value, Buffer&& buffer)
451		{
452		context ctx{};
453		return read<opts{.format = NDJSON}>(value, std::forward<Buffer>(buffer), ctx);
454		}
455
456		template <read_supported<NDJSON> T, class Buffer>
457		[[nodiscard]] expected<T, error_ctx> read_ndjson(Buffer&& buffer)
458		{
459		T value{};
460		context ctx{};
461		const auto ec = read<opts{.format = NDJSON}>(value, std::forward<Buffer>(buffer), ctx);
462		if (ec == error_code::none) {
463		return value;
464		}
465		return unexpected(ec);
466		}
467
468		template <auto Opts = opts{.format = NDJSON}, read_supported<NDJSON> T>
469		[[nodiscard]] error_ctx read_file_ndjson(T& value, const sv file_name)
470		{
471		context ctx{};
472		ctx.current_file = file_name;
473
474		std::string buffer;
475
476		const auto ec = file_to_buffer(buffer, ctx.current_file);
477
478		if (bool(ec)) {
479		return {0, ec};
480		}
481
482		return read<Opts>(value, buffer, ctx);
483		}
484
485		template <write_supported<NDJSON> T, class Buffer>
486		[[nodiscard]] error_ctx write_ndjson(T&& value, Buffer&& buffer)
487		{
488		return write<opts{.format = NDJSON}>(std::forward<T>(value), std::forward<Buffer>(buffer));
489		}
490
491		template <write_supported<NDJSON> T>
492		[[nodiscard]] expected<std::string, error_ctx> write_ndjson(T&& value)
493		{
494		return write<opts{.format = NDJSON}>(std::forward<T>(value));
495		}
496
497		template <write_supported<NDJSON> T>
498		[[nodiscard]] error_ctx write_file_ndjson(T&& value, const std::string& file_name, auto&& buffer)
499		{
500		const auto ec = write<opts{.format = NDJSON}>(std::forward<T>(value), buffer);
501		if (bool(ec)) [[unlikely]] {
502		return ec;
503		}
504		const auto file_ec = buffer_to_file(buffer, file_name);
505		if (bool(file_ec)) [[unlikely]] {
506		return {0, file_ec};
507		}
508		return {buffer.size(), error_code::none};
509		}
510		}