Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions mina-json-bytes/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,11 @@ mina-json-bytes encode < ledger.json | some-text-tool | mina-json-bytes decode >
```

`encode` replaces each byte that is not valid UTF-8 with the ASCII token
`__MINA_BYTE_xx__` (`xx`: the byte in lower-case hex). `decode` replaces each
token with its byte. All other bytes stay the same, so
`decode(encode(x)) == x`.
`__MINA_BYTE_xx__` (`xx`: the byte in lower-case hex). Where the input already
holds the text `__MINA_BYTE_`, `encode` also writes the first `_` of it as the
token `__MINA_BYTE_5f__`, so that `decode` cannot take that text for a token.
`decode` replaces each token with its byte. All other bytes stay the same, so
`decode(encode(x)) == x` for every input.

## Rules of the `jq` subcommand

Expand All @@ -52,8 +54,6 @@ mina-json-bytes jq 'del(.genesis)' < config.json > out.tmp && mv out.tmp out.jso

## Limits

- `encode` stops with an error if the input contains the text `__MINA_BYTE_`.
Without this rule, `decode` could not tell that text from a token.
- A jq filter sees the token and not the byte. For example, `length` or `test`
on such a field gives a result for the token.
- jq still changes the format of its output: whitespace, escapes such as
Expand Down
4 changes: 0 additions & 4 deletions mina-json-bytes/src/error.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
//! The errors of this tool.

use crate::token::Token;
use std::io;
use std::path::PathBuf;

Expand All @@ -21,9 +20,6 @@ pub enum UsageError {

#[derive(Debug, thiserror::Error)]
pub enum Error {
/// The encoder cannot keep this input apart from its own tokens.
#[error("the input already contains the text {}", Token::PREFIX)]
PrefixInInput,
#[error("{}: {source}", path.display())]
File { path: PathBuf, source: Box<Error> },
#[error("cannot start jq: {0}")]
Expand Down
6 changes: 3 additions & 3 deletions mina-json-bytes/src/token.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ use std::io::{self, Write};
pub(crate) struct Token;

impl Token {
/// The start of each token. The encoder rejects an input with this text.
/// Without it in the input, only a real token can match in the decoder,
/// also where a token touches the text around it. So
/// The start of each token. Where the input holds this text, the encoder
/// writes its first byte as a token. So the encoded text holds the prefix
/// only at a token, or where a token touches the text around it, and
/// `decode(encode(x)) == x`.
pub(crate) const PREFIX: &'static str = "__MINA_BYTE_";
const SUFFIX: &'static str = "__";
Expand Down
108 changes: 77 additions & 31 deletions mina-json-bytes/src/transcode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
//! A mainnet response is about 700 MB, so the data goes through in chunks and
//! only a short tail stays in memory between two reads.

use crate::error::{Error, Result};
use crate::error::Result;
use crate::token::Token;
use std::io::{self, BufWriter, Read, Write};

Expand Down Expand Up @@ -52,7 +52,8 @@ fn safe_end(pending: &[u8], at_end: bool, tail: usize) -> usize {
}
}

/// Replaces each byte that is not valid UTF-8 with a [`Token`].
/// Replaces each byte that is not valid UTF-8 with a [`Token`], and escapes
/// the text [`Token::PREFIX`] where the input already holds it.
pub struct Encoder<W: Write> {
writer: BufWriter<W>,
}
Expand All @@ -70,38 +71,56 @@ impl<W: Write> Encoder<W> {
.iter()
.try_for_each(|&byte| Token::write(&mut self.writer, byte))
}
}

impl<W: Write> Transcode for Encoder<W> {
fn convert(&mut self, pending: &[u8], at_end: bool) -> Result<usize> {
if Token::find_prefix(pending, 0).is_some() {
return Err(Error::PrefixInInput);
}
let end = safe_end(pending, at_end, Token::PREFIX.len() - 1);
/// Writes the valid UTF-8 of `text` as it is and each other byte as a
/// token, and returns the number of bytes it wrote. If `complete` is
/// false, a character cut at the end stays unwritten: a later read can
/// complete it.
fn write_text(&mut self, text: &[u8], complete: bool) -> io::Result<usize> {
let mut position = 0;
while position < end {
let error = match std::str::from_utf8(&pending[position..end]) {
while position < text.len() {
let error = match std::str::from_utf8(&text[position..]) {
Ok(_) => {
self.writer.write_all(&pending[position..end])?;
return Ok(end);
self.writer.write_all(&text[position..])?;
return Ok(text.len());
}
Err(error) => error,
};
let valid_end = position + error.valid_up_to();
self.writer.write_all(&pending[position..valid_end])?;
self.writer.write_all(&text[position..valid_end])?;
position = valid_end;
let invalid_end = match error.error_len() {
Some(length) => position + length,
// A character is cut at the end. A later read can complete
// it, and at the end of the input it is invalid.
None if at_end => end,
None if complete => text.len(),
None => break,
};
self.write_tokens(&pending[position..invalid_end])?;
self.write_tokens(&text[position..invalid_end])?;
position = invalid_end;
}
Ok(position)
}
}

impl<W: Write> Transcode for Encoder<W> {
fn convert(&mut self, pending: &[u8], at_end: bool) -> Result<usize> {
// A prefix that starts before this end is complete in `pending`.
let end = safe_end(pending, at_end, Token::PREFIX.len() - 1);
let mut position = 0;
while position < end {
let prefix = Token::find_prefix(pending, position).filter(|&start| start < end);
let Some(start) = prefix else {
// A character cut at the end of the input is invalid.
return Ok(position + self.write_text(&pending[position..end], at_end)?);
};
// The prefix follows, so a character cut here is invalid.
self.write_text(&pending[position..start], true)?;
// The first byte of the prefix as a token. The decoder gives the
// byte back, and the text after it is no longer a prefix.
Token::write(&mut self.writer, Token::PREFIX.as_bytes()[0])?;
position = start + 1;
}
Ok(position)
}

fn flush(&mut self) -> Result<()> {
Ok(self.writer.flush()?)
Expand Down Expand Up @@ -226,24 +245,52 @@ mod tests {
}

#[test]
fn encoder_rejects_the_prefix_in_the_input() {
let error = encoded(b"a __MINA_BYTE_ff__ b").unwrap_err();
assert!(matches!(error, Error::PrefixInInput));
fn encoder_escapes_the_prefix_in_the_input() {
let input = b"a __MINA_BYTE_ff__ b";
let output = encoded(input).unwrap();
assert_eq!(output, b"a __MINA_BYTE_5f___MINA_BYTE_ff__ b");
assert_eq!(decoded(&output), input);
// Also without a complete token: it can join with a real token.
assert!(encoded(b"__MINA_BYTE_ff\xff").is_err());
let input = b"__MINA_BYTE_ff\xff";
assert_eq!(decoded(&encoded(input).unwrap()), input);
}

#[test]
fn encoder_escapes_each_of_overlapping_prefixes() {
// The last underscore of one prefix is the first of the next.
for input in [
&b"__MINA_BYTE__MINA_BYTE_"[..],
b"___MINA_BYTE_",
b"__MINA_BYTE___MINA_BYTE_ff__",
b"__MINA_BYTE_5f__",
] {
let output = encoded(input).unwrap();
assert_eq!(decoded(&output), input, "input {input:?}");
}
}

#[test]
fn encoder_finds_the_prefix_across_a_chunk_border() {
let input = b"abc__MINA_BYTE_def";
let output = encoded(input).unwrap();
assert_eq!(output, b"abc__MINA_BYTE_5f___MINA_BYTE_def");
for size in 1..=Token::PREFIX.len() + 1 {
let reader = Trickle {
data: b"abc__MINA_BYTE_def",
size,
};
assert!(encoded_from(reader).is_err(), "read size {size}");
let reader = Trickle { data: input, size };
assert_eq!(encoded_from(reader).unwrap(), output, "read size {size}");
}
}

#[test]
fn cut_character_before_a_prefix_is_invalid() {
let input = b"\xe2\x82__MINA_BYTE_";
let output = encoded(input).unwrap();
assert_eq!(
output,
b"__MINA_BYTE_e2____MINA_BYTE_82____MINA_BYTE_5f___MINA_BYTE_"
);
assert_eq!(decoded(&output), input);
}

#[test]
fn decoder_leaves_text_that_is_not_a_token() {
for text in [
Expand Down Expand Up @@ -281,6 +328,8 @@ mod tests {
b"__MINA",
b"__MINA_BYTE",
b"MINA_BYTE_",
b"__MINA_BYTE_",
b"__MINA_BYTE_ff__",
b"ff",
b"f",
b"\xff",
Expand All @@ -304,10 +353,7 @@ mod tests {
for _ in 0..next() % 12 {
input.extend_from_slice(parts[next() % parts.len()]);
}
let Ok(output) = encoded(&input) else {
assert!(Token::find_prefix(&input, 0).is_some());
continue;
};
let output = encoded(&input).unwrap();
assert!(std::str::from_utf8(&output).is_ok());
assert_eq!(decoded(&output), input, "input {input:?}");
for size in [1, 2, 3, 7] {
Expand Down
23 changes: 15 additions & 8 deletions mina-json-bytes/tests/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,18 @@ fn encode_then_decode_returns_the_input() {
assert_eq!(decoded.stdout, LEDGER);
}

/// A zkApp URI is free text, so it can hold the text of a token.
const URI_WITH_A_TOKEN: &[u8] =
b"[{\"pk\":\"B62c\",\"token_symbol\":\"\xff\",\"zkapp\":{\"zkapp_uri\":\"__MINA_BYTE_ff__\xff\"}}]";

#[test]
fn encode_fails_on_the_prefix_in_the_input() {
let output = run(&["encode"], b"\"__MINA_BYTE_ff__\"");
assert_eq!(output.status.code(), Some(1));
assert!(stderr(&output).contains("__MINA_BYTE_"));
fn encode_then_decode_keeps_the_text_of_a_token() {
let encoded = run(&["encode"], URI_WITH_A_TOKEN);
assert!(encoded.status.success(), "{}", stderr(&encoded));
assert!(std::str::from_utf8(&encoded.stdout).is_ok());
let decoded = run(&["decode"], &encoded.stdout);
assert!(decoded.status.success());
assert_eq!(decoded.stdout, URI_WITH_A_TOKEN);
}

#[test]
Expand Down Expand Up @@ -169,8 +176,8 @@ fn missing_slurpfile_is_a_failure() {
}

#[test]
fn jq_fails_on_the_prefix_in_stdin() {
let output = run(&["jq", "."], b"\"__MINA_BYTE_ff__\"");
assert_eq!(output.status.code(), Some(1));
assert!(stderr(&output).contains("__MINA_BYTE_"));
fn jq_keeps_the_text_of_a_token() {
let output = run(&["jq", "-c", "."], URI_WITH_A_TOKEN);
assert!(output.status.success(), "{}", stderr(&output));
assert_eq!(output.stdout.strip_suffix(b"\n").unwrap(), URI_WITH_A_TOKEN);
}
Loading