From 7543d0e866c6eedb6f203c5954316e9b09fac714 Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 8 Dec 2022 05:16:14 +0900 Subject: [PATCH 01/15] use output_writer in more places --- src/main.rs | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/src/main.rs b/src/main.rs index e31a70e..c0861e1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,7 @@ mod json_borsh; #[derive(Parser, Debug)] #[command(author, version)] /// Command-line utility for manipulating Borsh-serialized data -/// +/// /// Note: Does not play particularly nicely with `HashMap<_, _>` types. struct Args { #[command(subcommand)] @@ -55,7 +55,7 @@ enum Command { }, /// Convert JSON to Borsh. /// - /// Note: Schemas are not yet supported, so values that can be null (e.g. a + /// Note: If a schema is not specified, values that can be null (e.g. a /// Rust Option), etc. WILL NOT be serialized correctly. Encode { /// Read input from this file if STDIN is empty. @@ -82,7 +82,7 @@ enum Command { #[arg(short, long)] output: Option, }, - /// Extracts the Borsh schema header + /// Extract the Borsh schema header. Extract { /// Read input from this file if STDIN is empty. #[arg(short, long)] @@ -92,7 +92,7 @@ enum Command { #[arg(short, long)] output: Option, }, - /// Removes the Borsh schema header + /// Remove the Borsh schema header. Strip { /// Read input from this file if STDIN is empty. #[arg(short, long)] @@ -141,19 +141,12 @@ fn output_borsh( } else { borsh::to_writer(writer, value) } - .expect("Failed to write Borsh to output"); + .expect("Failed to write Borsh"); } fn output_json(output: Option<&PathBuf>, value: &impl Serialize) { - if let Some(o) = output { - let path = o.display(); - let f = - fs::File::create(o).unwrap_or_else(|_| panic!("Could not create output file {path}")); - serde_json::to_writer(f, &value) - .unwrap_or_else(|_| panic!("Could not write JSON to output file {path}")); - } else { - serde_json::to_writer(io::stdout(), &value).expect("Could not write JSON to STDOUT"); - } + let writer = output_writer(output); + serde_json::to_writer(writer, value).expect("Failed to write JSON"); } fn main() { @@ -195,17 +188,8 @@ fn main() { v }; - if let Some(o) = output { - let mut f = fs::File::create(o) - .unwrap_or_else(|_| panic!("Could not create output file {}", o.display())); - - f.write_all(&value) - .unwrap_or_else(|_| panic!("Could not write to output file {}", o.display())); - } else { - io::stdout() - .write_all(&value) - .expect("Could not write to STDOUT"); - } + let mut writer = output_writer(output.as_ref()); + writer.write_all(&value).expect("Failed output"); } Command::Encode { input, @@ -262,7 +246,9 @@ fn main() { let _ = ::deserialize(&mut buf).unwrap(); - output_writer(output.as_ref()).write_all(buf).expect("Unable to write output"); + output_writer(output.as_ref()) + .write_all(buf) + .expect("Unable to write output"); } } } From a920d799ec7876d9d8a295511209d64b7d320f05 Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 8 Dec 2022 05:16:59 +0900 Subject: [PATCH 02/15] adds install section to README --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index d62fb4c..fb212e6 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ Command line utility for basic [Borsh](https://borsh.io/)-serialized data manipulations. +## Install + +```txt +$ cargo install borsh-cli +``` + ## Usage ```text From 4e0417eb4e6ad980bc6ab147961aadeae09838eb Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 8 Dec 2022 11:50:21 +0900 Subject: [PATCH 03/15] chore: run command / clean up --- src/main.rs | 250 ++++++++++++++++++++++++++-------------------------- 1 file changed, 124 insertions(+), 126 deletions(-) diff --git a/src/main.rs b/src/main.rs index c0861e1..3d82023 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,8 @@ mod json_borsh; #[command(author, version)] /// Command-line utility for manipulating Borsh-serialized data /// -/// Note: Does not play particularly nicely with `HashMap<_, _>` types. +/// Note: Does not play particularly nicely with `HashMap<_, _>` types in +/// schema. struct Args { #[command(subcommand)] command: Command, @@ -27,12 +28,10 @@ struct Args { enum Command { /// Serialize the input as a simple binary blob with Borsh headers. Pack { - /// Read input from this file if STDIN is empty. - #[arg(short, long)] + /// Read input from this file, otherwise from stdin. input: Option, - /// Write output this file, otherwise to STDOUT. - #[arg(short, long)] + /// Write output to this file, otherwise to stdout. output: Option, /// By default, the Borsh schema is included in the header. Enable this flag to remove it. @@ -41,12 +40,10 @@ enum Command { }, /// Deserialize the input as a simple binary blob with Borsh headers. Unpack { - /// Read input from this file if STDIN is empty. - #[arg(short, long)] + /// Read input from this file, otherwise from stdin. input: Option, - /// Write output this file, otherwise to STDOUT. - #[arg(short, long)] + /// Write output to this file, otherwise to stdout. output: Option, /// By default, we assume the Borsh schema is included in the header. Enable this flag to prevent this. @@ -58,12 +55,10 @@ enum Command { /// Note: If a schema is not specified, values that can be null (e.g. a /// Rust Option), etc. WILL NOT be serialized correctly. Encode { - /// Read input from this file if STDIN is empty. - #[arg(short, long)] + /// Read input from this file, otherwise from stdin. input: Option, - /// Write output this file, otherwise to STDOUT. - #[arg(short, long)] + /// Write output to this file, otherwise to stdout. output: Option, /// Schema to follow when serializing. @@ -74,36 +69,139 @@ enum Command { /// /// Requires the input to contain the embedded schema. Decode { - /// Read input from this file if STDIN is empty. - #[arg(short, long)] + /// Read input from this file, otherwise from stdin. input: Option, - /// Write output this file, otherwise to STDOUT. - #[arg(short, long)] + /// Write output to this file, otherwise to stdout. output: Option, }, /// Extract the Borsh schema header. Extract { - /// Read input from this file if STDIN is empty. - #[arg(short, long)] + /// Read input from this file, otherwise from stdin. input: Option, - /// Write output this file, otherwise to STDOUT. - #[arg(short, long)] + /// Write output to this file, otherwise to stdout. output: Option, }, /// Remove the Borsh schema header. Strip { - /// Read input from this file if STDIN is empty. - #[arg(short, long)] + /// Read input from this file, otherwise from stdin. input: Option, - /// Write output this file, otherwise to STDOUT. - #[arg(short, long)] + /// Write output to this file, otherwise to stdout. output: Option, }, } +impl Command { + fn run(&self) { + match self { + Command::Pack { + input, + output, + no_schema, + } => { + let input_bytes = get_input_bytes(input.as_ref()); + + let schema = Vec::::schema_container(); + + output_borsh( + output.as_ref(), + &input_bytes, + if *no_schema { None } else { Some(&schema) }, + ); + } + Command::Unpack { + input, + output, + no_schema, + } => { + let input_bytes = get_input_bytes(input.as_ref()); + + let value = if *no_schema { + Vec::::try_from_slice(&input_bytes) + .expect("Could not read input as byte array") + } else { + let (schema, v) = + <(BorshSchemaContainer, Vec)>::try_from_slice(&input_bytes) + .expect("Could not read input as byte array with schema headers"); + assert_eq!( + schema, + Vec::::schema_container(), + "Unexpected schema header: {}", + schema.declaration, + ); + v + }; + + let mut writer = output_writer(output.as_ref()); + writer.write_all(&value).expect("Failed output"); + } + Command::Encode { + input, + output, + schema, + } => { + let input_bytes = get_input_bytes(input.as_ref()); + + let v = serde_json::from_slice::(&input_bytes) + .expect("Could not parse input as JSON"); + + if let Some(schema_path) = schema { + let schema_bytes = get_input_bytes(Some(schema_path)); + let mut writer = output_writer(output.as_ref()); + let schema = ::deserialize( + &mut (&schema_bytes as &[u8]), + ) + .expect("Could not parse schema"); + BorshSerialize::serialize(&schema, &mut writer) + .expect("could not serialize schema to output"); + dynamic_schema::serialize_with_schema(&mut writer, &v, &schema) + .expect("Could not write output"); + } else { + let v = JsonSerializableAsBorsh(&v); + + output_borsh(output.as_ref(), &v, None); + } + } + Command::Decode { input, output } => { + let input_bytes = get_input_bytes(input.as_ref()); + + let mut buf = &input_bytes as &[u8]; + + let schema = + ::deserialize(&mut buf).unwrap(); + + let value = dynamic_schema::deserialize_from_schema(&mut buf, &schema) + .expect("Unable to deserialize according to embedded schema"); + + output_json(output.as_ref(), &value); + } + Command::Extract { input, output } => { + let input_bytes = get_input_bytes(input.as_ref()); + + let mut buf = &input_bytes as &[u8]; + + let schema = + ::deserialize(&mut buf).unwrap(); + + output_borsh(output.as_ref(), &schema, None); + } + Command::Strip { input, output } => { + let input_bytes = get_input_bytes(input.as_ref()); + + let mut buf = &input_bytes as &[u8]; + + let _ = ::deserialize(&mut buf).unwrap(); + + output_writer(output.as_ref()) + .write_all(buf) + .expect("Unable to write output"); + } + } + } +} + fn get_input_bytes(input_path: Option<&PathBuf>) -> Vec { input_path .map(|path| { @@ -150,107 +248,7 @@ fn output_json(output: Option<&PathBuf>, value: &impl Serialize) { } fn main() { - let args = Args::parse(); - - match &args.command { - Command::Pack { - input, - output, - no_schema, - } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let schema = Vec::::schema_container(); - - output_borsh( - output.as_ref(), - &input_bytes, - if *no_schema { None } else { Some(&schema) }, - ); - } - Command::Unpack { - input, - output, - no_schema, - } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let value = if *no_schema { - Vec::::try_from_slice(&input_bytes).expect("Could not read input as byte array") - } else { - let (schema, v) = <(BorshSchemaContainer, Vec)>::try_from_slice(&input_bytes) - .expect("Could not read input as byte array with schema headers"); - assert_eq!( - schema, - Vec::::schema_container(), - "Incorrect schema header", - ); - v - }; - - let mut writer = output_writer(output.as_ref()); - writer.write_all(&value).expect("Failed output"); - } - Command::Encode { - input, - output, - schema, - } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let v = serde_json::from_slice::(&input_bytes) - .expect("Could not parse input as JSON"); - - if let Some(schema_path) = schema { - let schema_bytes = get_input_bytes(Some(schema_path)); - let mut writer = output_writer(output.as_ref()); - let schema = ::deserialize( - &mut (&schema_bytes as &[u8]), - ) - .expect("Could not parse schema"); - BorshSerialize::serialize(&schema, &mut writer) - .expect("could not serialize schema to output"); - dynamic_schema::serialize_with_schema(&mut writer, &v, &schema) - .expect("Could not write output"); - } else { - let v = JsonSerializableAsBorsh(&v); - - output_borsh(output.as_ref(), &v, None); - } - } - Command::Decode { input, output } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let mut buf = &input_bytes as &[u8]; - - let schema = ::deserialize(&mut buf).unwrap(); - - let value = dynamic_schema::deserialize_from_schema(&mut buf, &schema) - .expect("Unable to deserialize according to embedded schema"); - - output_json(output.as_ref(), &value); - } - Command::Extract { input, output } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let mut buf = &input_bytes as &[u8]; - - let schema = ::deserialize(&mut buf).unwrap(); - - output_borsh(output.as_ref(), &schema, None); - } - Command::Strip { input, output } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let mut buf = &input_bytes as &[u8]; - - let _ = ::deserialize(&mut buf).unwrap(); - - output_writer(output.as_ref()) - .write_all(buf) - .expect("Unable to write output"); - } - } + Args::parse().command.run(); } #[cfg(test)] From 7d2ef590bcb65170ce575bf3e79c1eba92715142 Mon Sep 17 00:00:00 2001 From: Jacob Date: Wed, 14 Dec 2022 00:28:54 +0900 Subject: [PATCH 04/15] each command to its own file --- src/command/decode.rs | 15 ++ src/command/encode.rs | 20 +++ src/command/extract.rs | 13 ++ src/command/mod.rs | 286 ++++++++++++++++++++++++++++++++++ src/command/pack.rs | 45 ++++++ src/command/strip.rs | 13 ++ src/command/unpack.rs | 17 +++ src/dynamic_schema.rs | 4 +- src/main.rs | 339 +---------------------------------------- 9 files changed, 414 insertions(+), 338 deletions(-) create mode 100644 src/command/decode.rs create mode 100644 src/command/encode.rs create mode 100644 src/command/extract.rs create mode 100644 src/command/mod.rs create mode 100644 src/command/pack.rs create mode 100644 src/command/strip.rs create mode 100644 src/command/unpack.rs diff --git a/src/command/decode.rs b/src/command/decode.rs new file mode 100644 index 0000000..36ad3f7 --- /dev/null +++ b/src/command/decode.rs @@ -0,0 +1,15 @@ +use std::path::PathBuf; + +use clap::Args; + +#[derive(Args, Debug)] +/// Decode Borsh input to JSON. +/// +/// Requires the input to contain the embedded schema. +pub struct DecodeArgs { + /// Read input from this file, otherwise from stdin. + pub input: Option, + + /// Write output to this file, otherwise to stdout. + pub output: Option, +} diff --git a/src/command/encode.rs b/src/command/encode.rs new file mode 100644 index 0000000..20cb0cb --- /dev/null +++ b/src/command/encode.rs @@ -0,0 +1,20 @@ +use std::path::PathBuf; + +use clap::Args; + +#[derive(Args, Debug)] +/// Convert JSON to Borsh. +/// +/// Note: If a schema is not specified, values that can be null (e.g. a +/// Rust Option), etc. WILL NOT be serialized correctly. +pub struct EncodeArgs { + /// Read input from this file, otherwise from stdin. + pub input: Option, + + /// Write output to this file, otherwise to stdout. + pub output: Option, + + /// Schema to follow when serializing. + #[arg(short, long)] + pub schema: Option, +} diff --git a/src/command/extract.rs b/src/command/extract.rs new file mode 100644 index 0000000..51f3638 --- /dev/null +++ b/src/command/extract.rs @@ -0,0 +1,13 @@ +use std::path::PathBuf; + +use clap::Args; + +#[derive(Args, Debug)] +/// Extract the Borsh schema header. +pub struct ExtractArgs { + /// Read input from this file, otherwise from stdin. + pub input: Option, + + /// Write output to this file, otherwise to stdout. + pub output: Option, +} diff --git a/src/command/mod.rs b/src/command/mod.rs new file mode 100644 index 0000000..910f049 --- /dev/null +++ b/src/command/mod.rs @@ -0,0 +1,286 @@ +use std::{ + fs, + io::{self, Read, Write}, + path::PathBuf, +}; + +use borsh::{schema::BorshSchemaContainer, BorshDeserialize, BorshSchema, BorshSerialize}; +use clap::{Args, Subcommand}; +use serde::Serialize; + +use crate::json_borsh::JsonSerializableAsBorsh; + +use self::pack::Pack; + +mod decode; +mod encode; +mod extract; +mod pack; +mod strip; +mod unpack; + +pub(self) trait Execute { + fn execute(&mut self); +} + +#[derive(Subcommand, Debug)] +pub enum Command { + Pack(pack::PackArgs), + Unpack(unpack::UnpackArgs), + Encode(encode::EncodeArgs), + Decode(decode::DecodeArgs), + Extract(extract::ExtractArgs), + Strip(strip::StripArgs), +} + +impl Command { + pub fn run(&self) { + match self { + Command::Pack(args) => { + Pack::execute(&mut args.into()) + } + Command::Unpack(unpack::UnpackArgs { + input, + output, + no_schema, + }) => { + let input_bytes = get_input_bytes(input.as_ref()); + + let value = if *no_schema { + Vec::::try_from_slice(&input_bytes) + .expect("Could not read input as byte array") + } else { + let (schema, v) = + <(BorshSchemaContainer, Vec)>::try_from_slice(&input_bytes) + .expect("Could not read input as byte array with schema headers"); + assert_eq!( + schema, + Vec::::schema_container(), + "Unexpected schema header: {}", + schema.declaration, + ); + v + }; + + let mut writer = output_writer(output.as_ref()); + writer.write_all(&value).expect("Failed output"); + } + Command::Encode(encode::EncodeArgs { + input, + output, + schema, + }) => { + let input_bytes = get_input_bytes(input.as_ref()); + + let v = serde_json::from_slice::(&input_bytes) + .expect("Could not parse input as JSON"); + + if let Some(schema_path) = schema { + let schema_bytes = get_input_bytes(Some(schema_path)); + let mut writer = output_writer(output.as_ref()); + let schema = ::deserialize( + &mut (&schema_bytes as &[u8]), + ) + .expect("Could not parse schema"); + BorshSerialize::serialize(&schema, &mut writer) + .expect("could not serialize schema to output"); + crate::dynamic_schema::serialize_with_schema(&mut writer, &v, &schema) + .expect("Could not write output"); + } else { + let v = JsonSerializableAsBorsh(&v); + + output_borsh(output.as_ref(), &v, None); + } + } + Command::Decode(decode::DecodeArgs { input, output }) => { + let input_bytes = get_input_bytes(input.as_ref()); + + let mut buf = &input_bytes as &[u8]; + + let schema = + ::deserialize(&mut buf).unwrap(); + + let value = crate::dynamic_schema::deserialize_from_schema(&mut buf, &schema) + .expect("Unable to deserialize according to embedded schema"); + + output_json(output.as_ref(), &value); + } + Command::Extract(extract::ExtractArgs { input, output }) => { + let input_bytes = get_input_bytes(input.as_ref()); + + let mut buf = &input_bytes as &[u8]; + + let schema = + ::deserialize(&mut buf).unwrap(); + + output_borsh(output.as_ref(), &schema, None); + } + Command::Strip(strip::StripArgs { input, output }) => { + let input_bytes = get_input_bytes(input.as_ref()); + + let mut buf = &input_bytes as &[u8]; + + let _ = ::deserialize(&mut buf).unwrap(); + + output_writer(output.as_ref()) + .write_all(buf) + .expect("Unable to write output"); + } + } + } +} + +fn get_input_bytes(input_path: Option<&PathBuf>) -> Vec { + input_path + .map(|path| { + fs::read(path) + .unwrap_or_else(|_| panic!("Could not read input file {}", path.display())) + }) + .unwrap_or_else(read_stdin) +} + +fn read_stdin() -> Vec { + let mut v = Vec::new(); + io::stdin() + .read_to_end(&mut v) + .expect("Could not read from STDIN"); + v +} + +fn output_writer(output: Option<&PathBuf>) -> Box { + if let Some(o) = output { + let f = fs::File::create(o) + .unwrap_or_else(|_| panic!("Could not create output file {}", o.display())); + Box::new(f) as Box + } else { + Box::new(io::stdout()) as Box + } +} + +fn output_borsh2( + writer: impl Write, + value: impl BorshSerialize, +) { + borsh::to_writer(writer, &value).expect("Failed to write Borsh"); +} + +fn output_borsh( + output: Option<&PathBuf>, + value: &impl BorshSerialize, + schema: Option<&BorshSchemaContainer>, +) { + let writer = output_writer(output); + + if let Some(schema) = schema { + borsh::to_writer(writer, &(schema, value)) + } else { + borsh::to_writer(writer, value) + } + .expect("Failed to write Borsh"); +} + +fn output_json(output: Option<&PathBuf>, value: &impl Serialize) { + let writer = output_writer(output); + serde_json::to_writer(writer, value).expect("Failed to write JSON"); +} + +#[cfg(test)] +#[allow(unused, dead_code)] +mod tests { + use borsh::{BorshSchema, BorshSerialize}; + use serde::Serialize; + + use super::{output_borsh, output_json}; + + #[test] + fn test_schema() { + #[derive(BorshSerialize, BorshSchema, Serialize)] + struct First { + a: (u32, u64), + b: String, + c: Second, + // d: HashMap, + e: Vec, + } + + #[derive(BorshSerialize, BorshSchema, Serialize)] + struct Second { + a: Third, + b: Third, + c: Third, + d: u32, + e: u32, + } + + #[derive(BorshSerialize, BorshSchema, Serialize)] + enum Third { + Alpha { field: u32 }, + Beta(u32), + Gamma, + } + + dbg!("{:?}", First::schema_container()); + // return; + let v = First { + a: (32, 64), + b: "String".to_string(), + c: Second { + a: Third::Alpha { field: 1 }, + b: Third::Beta(1), + c: Third::Gamma, + d: 2, + e: 3, + }, + // d: vec![("true".to_string(), true), ("false".to_string(), false)] + // .into_iter() + // .collect(), + e: vec!["a".to_string(), "b".to_string(), "c".to_string()], + }; + borsh::try_to_vec_with_schema(&v); + output_json( + Some(&"./dataonly.json".into()), + &v, + // Some(&First::schema_container()), + ); + output_borsh( + Some(&"./dataandschema.borsh".into()), + &v, + Some(&First::schema_container()), + ); + } + + #[test] + fn ensure_key_order_is_preserved() { + let s1 = r#"{ + "a": 1, + "b": 1, + "c": 1, + "d": 1, + "e": 1, + "f": 1 + }"#; + let s2 = r#"{ + "f": 1, + "e": 1, + "d": 1, + "c": 1, + "b": 1, + "a": 1 + }"#; + + let v1 = serde_json::from_str::(s1).unwrap(); + let v2 = serde_json::from_str::(s2).unwrap(); + + assert_eq!(v1, v2); + + let (o1, o2) = match (v1, v2) { + (serde_json::Value::Object(o1), serde_json::Value::Object(o2)) => (o1, o2), + _ => unreachable!(), + }; + + let k1 = o1.keys().collect::>(); + let k2 = o2.keys().collect::>(); + + assert_ne!(k1, k2); + } +} diff --git a/src/command/pack.rs b/src/command/pack.rs new file mode 100644 index 0000000..f12fc0d --- /dev/null +++ b/src/command/pack.rs @@ -0,0 +1,45 @@ +use std::{io::Write, path::PathBuf}; + +use borsh::BorshSchema; +use clap::Args; + +#[derive(Args, Debug)] +/// Serialize the input as a simple binary blob with Borsh headers. +pub struct PackArgs { + /// Read input from this file, otherwise from stdin. + pub input_path: Option, + + /// Write output to this file, otherwise to stdout. + pub output_path: Option, + + /// By default, the Borsh schema is included in the header. Enable this flag to remove it. + #[arg(short, long)] + pub no_schema: bool, +} + +pub struct Pack { + pub input: Vec, + pub output: Box, + pub no_schema: bool, +} + +impl From<&'_ PackArgs> for Pack { + fn from(args: &'_ PackArgs) -> Self { + Self { + input: super::get_input_bytes(args.input_path.as_ref()), + output: super::output_writer(args.output_path.as_ref()), + no_schema: args.no_schema, + } + } +} + +impl super::Execute for Pack { + fn execute(&mut self) { + if self.no_schema { + super::output_borsh2(&mut self.output, &self.input); + } else { + let schema = Vec::::schema_container(); + super::output_borsh2(&mut self.output, &(schema, &self.input)); + } + } +} diff --git a/src/command/strip.rs b/src/command/strip.rs new file mode 100644 index 0000000..0337f56 --- /dev/null +++ b/src/command/strip.rs @@ -0,0 +1,13 @@ +use std::path::PathBuf; + +use clap::Args; + +#[derive(Args, Debug)] +/// Remove the Borsh schema header. +pub struct StripArgs { + /// Read input from this file, otherwise from stdin. + pub input: Option, + + /// Write output to this file, otherwise to stdout. + pub output: Option, +} diff --git a/src/command/unpack.rs b/src/command/unpack.rs new file mode 100644 index 0000000..1357b2a --- /dev/null +++ b/src/command/unpack.rs @@ -0,0 +1,17 @@ +use std::path::PathBuf; + +use clap::Args; + +#[derive(Args, Debug)] +/// Deserialize the input as a simple binary blob with Borsh headers. +pub struct UnpackArgs { + /// Read input from this file, otherwise from stdin. + pub input: Option, + + /// Write output to this file, otherwise to stdout. + pub output: Option, + + /// By default, we assume the Borsh schema is included in the header. Enable this flag to prevent this. + #[arg(short, long)] + pub no_schema: bool, +} diff --git a/src/dynamic_schema.rs b/src/dynamic_schema.rs index 28e3ab9..cfc2aa7 100644 --- a/src/dynamic_schema.rs +++ b/src/dynamic_schema.rs @@ -17,7 +17,7 @@ fn deserialize_type>( .map_err(|_| Error::new(std::io::ErrorKind::InvalidData, type_name)) } -pub fn deserialize_declaration_from_schema( +fn deserialize_declaration_from_schema( buf: &mut &[u8], schema: &BorshSchemaContainer, declaration: &borsh::schema::Declaration, @@ -190,7 +190,7 @@ where Ok(()) } -pub fn serialize_declaration_with_schema( +fn serialize_declaration_with_schema( writer: &mut impl Write, value: &serde_json::Value, schema: &BorshSchemaContainer, diff --git a/src/main.rs b/src/main.rs index 3d82023..af8439e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,15 +1,6 @@ -use std::{ - fs, - io::{self, Read, Write}, - path::PathBuf, -}; - -use borsh::{schema::BorshSchemaContainer, BorshDeserialize, BorshSchema, BorshSerialize}; -use clap::{Parser, Subcommand}; -use serde::Serialize; - -use crate::json_borsh::JsonSerializableAsBorsh; +use clap::Parser; +mod command; mod dynamic_schema; mod json_borsh; @@ -21,333 +12,9 @@ mod json_borsh; /// schema. struct Args { #[command(subcommand)] - command: Command, -} - -#[derive(Subcommand, Debug)] -enum Command { - /// Serialize the input as a simple binary blob with Borsh headers. - Pack { - /// Read input from this file, otherwise from stdin. - input: Option, - - /// Write output to this file, otherwise to stdout. - output: Option, - - /// By default, the Borsh schema is included in the header. Enable this flag to remove it. - #[arg(short, long)] - no_schema: bool, - }, - /// Deserialize the input as a simple binary blob with Borsh headers. - Unpack { - /// Read input from this file, otherwise from stdin. - input: Option, - - /// Write output to this file, otherwise to stdout. - output: Option, - - /// By default, we assume the Borsh schema is included in the header. Enable this flag to prevent this. - #[arg(short, long)] - no_schema: bool, - }, - /// Convert JSON to Borsh. - /// - /// Note: If a schema is not specified, values that can be null (e.g. a - /// Rust Option), etc. WILL NOT be serialized correctly. - Encode { - /// Read input from this file, otherwise from stdin. - input: Option, - - /// Write output to this file, otherwise to stdout. - output: Option, - - /// Schema to follow when serializing. - #[arg(short, long)] - schema: Option, - }, - /// Decode Borsh input to JSON. - /// - /// Requires the input to contain the embedded schema. - Decode { - /// Read input from this file, otherwise from stdin. - input: Option, - - /// Write output to this file, otherwise to stdout. - output: Option, - }, - /// Extract the Borsh schema header. - Extract { - /// Read input from this file, otherwise from stdin. - input: Option, - - /// Write output to this file, otherwise to stdout. - output: Option, - }, - /// Remove the Borsh schema header. - Strip { - /// Read input from this file, otherwise from stdin. - input: Option, - - /// Write output to this file, otherwise to stdout. - output: Option, - }, -} - -impl Command { - fn run(&self) { - match self { - Command::Pack { - input, - output, - no_schema, - } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let schema = Vec::::schema_container(); - - output_borsh( - output.as_ref(), - &input_bytes, - if *no_schema { None } else { Some(&schema) }, - ); - } - Command::Unpack { - input, - output, - no_schema, - } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let value = if *no_schema { - Vec::::try_from_slice(&input_bytes) - .expect("Could not read input as byte array") - } else { - let (schema, v) = - <(BorshSchemaContainer, Vec)>::try_from_slice(&input_bytes) - .expect("Could not read input as byte array with schema headers"); - assert_eq!( - schema, - Vec::::schema_container(), - "Unexpected schema header: {}", - schema.declaration, - ); - v - }; - - let mut writer = output_writer(output.as_ref()); - writer.write_all(&value).expect("Failed output"); - } - Command::Encode { - input, - output, - schema, - } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let v = serde_json::from_slice::(&input_bytes) - .expect("Could not parse input as JSON"); - - if let Some(schema_path) = schema { - let schema_bytes = get_input_bytes(Some(schema_path)); - let mut writer = output_writer(output.as_ref()); - let schema = ::deserialize( - &mut (&schema_bytes as &[u8]), - ) - .expect("Could not parse schema"); - BorshSerialize::serialize(&schema, &mut writer) - .expect("could not serialize schema to output"); - dynamic_schema::serialize_with_schema(&mut writer, &v, &schema) - .expect("Could not write output"); - } else { - let v = JsonSerializableAsBorsh(&v); - - output_borsh(output.as_ref(), &v, None); - } - } - Command::Decode { input, output } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let mut buf = &input_bytes as &[u8]; - - let schema = - ::deserialize(&mut buf).unwrap(); - - let value = dynamic_schema::deserialize_from_schema(&mut buf, &schema) - .expect("Unable to deserialize according to embedded schema"); - - output_json(output.as_ref(), &value); - } - Command::Extract { input, output } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let mut buf = &input_bytes as &[u8]; - - let schema = - ::deserialize(&mut buf).unwrap(); - - output_borsh(output.as_ref(), &schema, None); - } - Command::Strip { input, output } => { - let input_bytes = get_input_bytes(input.as_ref()); - - let mut buf = &input_bytes as &[u8]; - - let _ = ::deserialize(&mut buf).unwrap(); - - output_writer(output.as_ref()) - .write_all(buf) - .expect("Unable to write output"); - } - } - } -} - -fn get_input_bytes(input_path: Option<&PathBuf>) -> Vec { - input_path - .map(|path| { - fs::read(path) - .unwrap_or_else(|_| panic!("Could not read input file {}", path.display())) - }) - .unwrap_or_else(|| { - let mut v = Vec::new(); - io::stdin() - .read_to_end(&mut v) - .expect("Could not read from STDIN"); - v - }) -} - -fn output_writer(output: Option<&PathBuf>) -> Box { - if let Some(o) = output { - let f = fs::File::create(o) - .unwrap_or_else(|_| panic!("Could not create output file {}", o.display())); - Box::new(f) as Box - } else { - Box::new(io::stdout()) as Box - } -} - -fn output_borsh( - output: Option<&PathBuf>, - value: &impl BorshSerialize, - schema: Option<&BorshSchemaContainer>, -) { - let writer = output_writer(output); - - if let Some(schema) = schema { - borsh::to_writer(writer, &(schema, value)) - } else { - borsh::to_writer(writer, value) - } - .expect("Failed to write Borsh"); -} - -fn output_json(output: Option<&PathBuf>, value: &impl Serialize) { - let writer = output_writer(output); - serde_json::to_writer(writer, value).expect("Failed to write JSON"); + command: command::Command, } fn main() { Args::parse().command.run(); } - -#[cfg(test)] -#[allow(unused, dead_code)] -mod tests { - use borsh::{BorshSchema, BorshSerialize}; - use serde::Serialize; - - use crate::{output_borsh, output_json}; - - #[test] - fn test_schema() { - #[derive(BorshSerialize, BorshSchema, Serialize)] - struct First { - a: (u32, u64), - b: String, - c: Second, - // d: HashMap, - e: Vec, - } - - #[derive(BorshSerialize, BorshSchema, Serialize)] - struct Second { - a: Third, - b: Third, - c: Third, - d: u32, - e: u32, - } - - #[derive(BorshSerialize, BorshSchema, Serialize)] - enum Third { - Alpha { field: u32 }, - Beta(u32), - Gamma, - } - - dbg!("{:?}", First::schema_container()); - // return; - let v = First { - a: (32, 64), - b: "String".to_string(), - c: Second { - a: Third::Alpha { field: 1 }, - b: Third::Beta(1), - c: Third::Gamma, - d: 2, - e: 3, - }, - // d: vec![("true".to_string(), true), ("false".to_string(), false)] - // .into_iter() - // .collect(), - e: vec!["a".to_string(), "b".to_string(), "c".to_string()], - }; - borsh::try_to_vec_with_schema(&v); - output_json( - Some(&"./dataonly.json".into()), - &v, - // Some(&First::schema_container()), - ); - output_borsh( - Some(&"./dataandschema.borsh".into()), - &v, - Some(&First::schema_container()), - ); - } - - #[test] - fn ensure_key_order_is_preserved() { - let s1 = r#"{ - "a": 1, - "b": 1, - "c": 1, - "d": 1, - "e": 1, - "f": 1 - }"#; - let s2 = r#"{ - "f": 1, - "e": 1, - "d": 1, - "c": 1, - "b": 1, - "a": 1 - }"#; - - let v1 = serde_json::from_str::(s1).unwrap(); - let v2 = serde_json::from_str::(s2).unwrap(); - - assert_eq!(v1, v2); - - let (o1, o2) = match (v1, v2) { - (serde_json::Value::Object(o1), serde_json::Value::Object(o2)) => (o1, o2), - _ => unreachable!(), - }; - - let k1 = o1.keys().collect::>(); - let k2 = o2.keys().collect::>(); - - assert_ne!(k1, k2); - } -} From 89fd3a1b1b82be5f455e2cc53e022c2d68e8f53b Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 15 Dec 2022 13:03:35 +0900 Subject: [PATCH 05/15] unpack own file --- src/command/mod.rs | 115 ++++++++++++++++++++---------------------- src/command/pack.rs | 20 ++++---- src/command/unpack.rs | 44 ++++++++++++++-- 3 files changed, 106 insertions(+), 73 deletions(-) diff --git a/src/command/mod.rs b/src/command/mod.rs index 910f049..ece48dd 100644 --- a/src/command/mod.rs +++ b/src/command/mod.rs @@ -7,10 +7,11 @@ use std::{ use borsh::{schema::BorshSchemaContainer, BorshDeserialize, BorshSchema, BorshSerialize}; use clap::{Args, Subcommand}; use serde::Serialize; +use thiserror::Error; use crate::json_borsh::JsonSerializableAsBorsh; -use self::pack::Pack; +use self::{pack::Pack, unpack::Unpack}; mod decode; mod encode; @@ -20,7 +21,7 @@ mod strip; mod unpack; pub(self) trait Execute { - fn execute(&mut self); + fn execute(&mut self) -> Result<(), IOError>; } #[derive(Subcommand, Debug)] @@ -36,48 +37,21 @@ pub enum Command { impl Command { pub fn run(&self) { match self { - Command::Pack(args) => { - Pack::execute(&mut args.into()) - } - Command::Unpack(unpack::UnpackArgs { - input, - output, - no_schema, - }) => { - let input_bytes = get_input_bytes(input.as_ref()); - - let value = if *no_schema { - Vec::::try_from_slice(&input_bytes) - .expect("Could not read input as byte array") - } else { - let (schema, v) = - <(BorshSchemaContainer, Vec)>::try_from_slice(&input_bytes) - .expect("Could not read input as byte array with schema headers"); - assert_eq!( - schema, - Vec::::schema_container(), - "Unexpected schema header: {}", - schema.declaration, - ); - v - }; - - let mut writer = output_writer(output.as_ref()); - writer.write_all(&value).expect("Failed output"); - } + Command::Pack(args) => Pack::execute(&mut args.try_into().unwrap()).unwrap(), + Command::Unpack(args) => Unpack::execute(&mut args.try_into().unwrap()).unwrap(), Command::Encode(encode::EncodeArgs { input, output, schema, }) => { - let input_bytes = get_input_bytes(input.as_ref()); + let input_bytes = get_input_bytes(input.as_ref()).unwrap(); let v = serde_json::from_slice::(&input_bytes) .expect("Could not parse input as JSON"); if let Some(schema_path) = schema { - let schema_bytes = get_input_bytes(Some(schema_path)); - let mut writer = output_writer(output.as_ref()); + let schema_bytes = get_input_bytes(Some(schema_path)).unwrap(); + let mut writer = output_writer(output.as_ref()).unwrap(); let schema = ::deserialize( &mut (&schema_bytes as &[u8]), ) @@ -93,7 +67,7 @@ impl Command { } } Command::Decode(decode::DecodeArgs { input, output }) => { - let input_bytes = get_input_bytes(input.as_ref()); + let input_bytes = get_input_bytes(input.as_ref()).unwrap(); let mut buf = &input_bytes as &[u8]; @@ -106,7 +80,7 @@ impl Command { output_json(output.as_ref(), &value); } Command::Extract(extract::ExtractArgs { input, output }) => { - let input_bytes = get_input_bytes(input.as_ref()); + let input_bytes = get_input_bytes(input.as_ref()).unwrap(); let mut buf = &input_bytes as &[u8]; @@ -116,13 +90,14 @@ impl Command { output_borsh(output.as_ref(), &schema, None); } Command::Strip(strip::StripArgs { input, output }) => { - let input_bytes = get_input_bytes(input.as_ref()); + let input_bytes = get_input_bytes(input.as_ref()).unwrap(); let mut buf = &input_bytes as &[u8]; let _ = ::deserialize(&mut buf).unwrap(); output_writer(output.as_ref()) + .unwrap() .write_all(buf) .expect("Unable to write output"); } @@ -130,38 +105,56 @@ impl Command { } } -fn get_input_bytes(input_path: Option<&PathBuf>) -> Vec { - input_path - .map(|path| { - fs::read(path) - .unwrap_or_else(|_| panic!("Could not read input file {}", path.display())) - }) - .unwrap_or_else(read_stdin) +#[derive(Error, Debug)] +pub enum IOError { + #[error("Failed to read input file {0}")] + ReadInputFile(String), + #[error("Failed to read from STDIN")] + ReadStdin, + #[error("Failed to create output file {0}")] + CreateOutputFile(String), + #[error("Failed to write Borsh")] + WriteBorsh, + #[error("Failed to write JSON")] + WriteJson, + #[error("Failed to write raw bytes")] + WriteBytes, + #[error("Failed to deserialize input as Borsh {0}")] + DeserializeBorsh(&'static str), + #[error("Unexpected schema header: {0}")] + IncorrectBorshSchemaHeader(String), +} + +fn get_input_bytes(input_path: Option<&PathBuf>) -> Result, IOError> { + input_path.map_or_else(read_stdin, |path| { + fs::read(path).map_err(|_| IOError::ReadInputFile(path.display().to_string())) + }) } -fn read_stdin() -> Vec { +fn read_stdin() -> Result, IOError> { let mut v = Vec::new(); io::stdin() .read_to_end(&mut v) - .expect("Could not read from STDIN"); - v + .map_err(|_e| IOError::ReadStdin)?; + Ok(v) } -fn output_writer(output: Option<&PathBuf>) -> Box { +fn output_writer(output: Option<&PathBuf>) -> Result, IOError> { if let Some(o) = output { - let f = fs::File::create(o) - .unwrap_or_else(|_| panic!("Could not create output file {}", o.display())); - Box::new(f) as Box + let f = + fs::File::create(o).map_err(|e| IOError::CreateOutputFile(o.display().to_string()))?; + Ok(Box::new(f) as Box) } else { - Box::new(io::stdout()) as Box + Ok(Box::new(io::stdout()) as Box) } } -fn output_borsh2( - writer: impl Write, - value: impl BorshSerialize, -) { - borsh::to_writer(writer, &value).expect("Failed to write Borsh"); +fn output_bytes(writer: &mut impl Write, value: &[u8]) -> Result<(), IOError> { + writer.write_all(value).map_err(|_| IOError::WriteBytes) +} + +fn output_borsh2(writer: impl Write, value: impl BorshSerialize) -> Result<(), IOError> { + borsh::to_writer(writer, &value).map_err(|_| IOError::WriteBorsh) } fn output_borsh( @@ -169,7 +162,7 @@ fn output_borsh( value: &impl BorshSerialize, schema: Option<&BorshSchemaContainer>, ) { - let writer = output_writer(output); + let writer = output_writer(output).unwrap(); if let Some(schema) = schema { borsh::to_writer(writer, &(schema, value)) @@ -179,9 +172,9 @@ fn output_borsh( .expect("Failed to write Borsh"); } -fn output_json(output: Option<&PathBuf>, value: &impl Serialize) { - let writer = output_writer(output); - serde_json::to_writer(writer, value).expect("Failed to write JSON"); +fn output_json(output: Option<&PathBuf>, value: &impl Serialize) -> Result<(), IOError> { + let writer = output_writer(output)?; + serde_json::to_writer(writer, value).map_err(|_| IOError::WriteJson) } #[cfg(test)] diff --git a/src/command/pack.rs b/src/command/pack.rs index f12fc0d..3a978aa 100644 --- a/src/command/pack.rs +++ b/src/command/pack.rs @@ -23,23 +23,25 @@ pub struct Pack { pub no_schema: bool, } -impl From<&'_ PackArgs> for Pack { - fn from(args: &'_ PackArgs) -> Self { - Self { - input: super::get_input_bytes(args.input_path.as_ref()), - output: super::output_writer(args.output_path.as_ref()), +impl TryFrom<&'_ PackArgs> for Pack { + type Error = super::IOError; + + fn try_from(args: &'_ PackArgs) -> Result { + Ok(Self { + input: super::get_input_bytes(args.input_path.as_ref())?, + output: super::output_writer(args.output_path.as_ref())?, no_schema: args.no_schema, - } + }) } } impl super::Execute for Pack { - fn execute(&mut self) { + fn execute(&mut self) -> Result<(), super::IOError> { if self.no_schema { - super::output_borsh2(&mut self.output, &self.input); + super::output_borsh2(&mut self.output, &self.input) } else { let schema = Vec::::schema_container(); - super::output_borsh2(&mut self.output, &(schema, &self.input)); + super::output_borsh2(&mut self.output, &(schema, &self.input)) } } } diff --git a/src/command/unpack.rs b/src/command/unpack.rs index 1357b2a..c4f9244 100644 --- a/src/command/unpack.rs +++ b/src/command/unpack.rs @@ -1,17 +1,55 @@ -use std::path::PathBuf; +use std::{io::Write, path::PathBuf}; +use borsh::{schema::BorshSchemaContainer, BorshDeserialize, BorshSchema}; use clap::Args; +use super::{output_bytes, IOError}; + #[derive(Args, Debug)] /// Deserialize the input as a simple binary blob with Borsh headers. pub struct UnpackArgs { /// Read input from this file, otherwise from stdin. - pub input: Option, + pub input_path: Option, /// Write output to this file, otherwise to stdout. - pub output: Option, + pub output_path: Option, /// By default, we assume the Borsh schema is included in the header. Enable this flag to prevent this. #[arg(short, long)] pub no_schema: bool, } + +impl TryFrom<&'_ UnpackArgs> for Unpack { + type Error = super::IOError; + fn try_from(args: &'_ UnpackArgs) -> Result { + Ok(Self { + input: super::get_input_bytes(args.input_path.as_ref())?, + output: super::output_writer(args.output_path.as_ref())?, + no_schema: args.no_schema, + }) + } +} + +pub struct Unpack { + pub input: Vec, + pub output: Box, + pub no_schema: bool, +} + +impl super::Execute for Unpack { + fn execute(&mut self) -> Result<(), super::IOError> { + let value = if self.no_schema { + Vec::::try_from_slice(&self.input) + .map_err(|_| IOError::DeserializeBorsh("byte array"))? + } else { + let (schema, v) = <(BorshSchemaContainer, Vec)>::try_from_slice(&self.input) + .map_err(|_| IOError::DeserializeBorsh("byte array with schema headers"))?; + if schema != Vec::::schema_container() { + return Err(IOError::IncorrectBorshSchemaHeader(schema.declaration)); + } + v + }; + + output_bytes(&mut self.output, &value) + } +} From e717d3df3480928e11f10d711865e87dd545fbf1 Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 15 Dec 2022 13:59:32 +0900 Subject: [PATCH 06/15] encode own file --- src/command/encode.rs | 53 ++++++++++++++++++++++++++++++++++++++++--- src/command/mod.rs | 36 +++++------------------------ 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/src/command/encode.rs b/src/command/encode.rs index 20cb0cb..d43a65c 100644 --- a/src/command/encode.rs +++ b/src/command/encode.rs @@ -1,7 +1,12 @@ -use std::path::PathBuf; +use std::{io::Write, path::PathBuf}; +use borsh::{schema::BorshSchemaContainer, BorshDeserialize, BorshSerialize}; use clap::Args; +use crate::{dynamic_schema::serialize_with_schema, json_borsh::JsonSerializableAsBorsh}; + +use super::{get_input_bytes, output_borsh2, output_writer, Execute, IOError}; + #[derive(Args, Debug)] /// Convert JSON to Borsh. /// @@ -9,12 +14,54 @@ use clap::Args; /// Rust Option), etc. WILL NOT be serialized correctly. pub struct EncodeArgs { /// Read input from this file, otherwise from stdin. - pub input: Option, + pub input_path: Option, /// Write output to this file, otherwise to stdout. - pub output: Option, + pub output_path: Option, /// Schema to follow when serializing. #[arg(short, long)] pub schema: Option, } + +pub struct Encode { + pub input: serde_json::Value, + pub output: Box, + pub schema: Option, +} + +impl TryFrom<&'_ EncodeArgs> for Encode { + type Error = IOError; + + fn try_from(args: &'_ EncodeArgs) -> Result { + Ok(Self { + input: serde_json::from_slice(&get_input_bytes(args.input_path.as_ref())?) + .map_err(|_e| IOError::DeserializeJson)?, + output: output_writer(args.output_path.as_ref())?, + schema: if let Some(ref path) = args.schema { + let schema_bytes = get_input_bytes(Some(path))?; + let schema = ::deserialize( + &mut (&schema_bytes as &[u8]), + ) + .map_err(|_| IOError::DeserializeBorsh("schema header"))?; + + Some(schema) + } else { + None + }, + }) + } +} + +impl Execute for Encode { + fn execute(&mut self) -> Result<(), IOError> { + let writer = &mut self.output; + if let Some(schema) = &self.schema { + BorshSerialize::serialize(&schema, writer).map_err(|_| IOError::WriteBorsh)?; + serialize_with_schema(writer, &self.input, &schema).map_err(|_| IOError::WriteBorsh)?; + Ok(()) + } else { + output_borsh2(writer, &JsonSerializableAsBorsh(&self.input)) + } + } +} diff --git a/src/command/mod.rs b/src/command/mod.rs index ece48dd..e994a7f 100644 --- a/src/command/mod.rs +++ b/src/command/mod.rs @@ -11,7 +11,7 @@ use thiserror::Error; use crate::json_borsh::JsonSerializableAsBorsh; -use self::{pack::Pack, unpack::Unpack}; +use self::{encode::Encode, pack::Pack, unpack::Unpack}; mod decode; mod encode; @@ -39,33 +39,7 @@ impl Command { match self { Command::Pack(args) => Pack::execute(&mut args.try_into().unwrap()).unwrap(), Command::Unpack(args) => Unpack::execute(&mut args.try_into().unwrap()).unwrap(), - Command::Encode(encode::EncodeArgs { - input, - output, - schema, - }) => { - let input_bytes = get_input_bytes(input.as_ref()).unwrap(); - - let v = serde_json::from_slice::(&input_bytes) - .expect("Could not parse input as JSON"); - - if let Some(schema_path) = schema { - let schema_bytes = get_input_bytes(Some(schema_path)).unwrap(); - let mut writer = output_writer(output.as_ref()).unwrap(); - let schema = ::deserialize( - &mut (&schema_bytes as &[u8]), - ) - .expect("Could not parse schema"); - BorshSerialize::serialize(&schema, &mut writer) - .expect("could not serialize schema to output"); - crate::dynamic_schema::serialize_with_schema(&mut writer, &v, &schema) - .expect("Could not write output"); - } else { - let v = JsonSerializableAsBorsh(&v); - - output_borsh(output.as_ref(), &v, None); - } - } + Command::Encode(args) => Encode::execute(&mut args.try_into().unwrap()).unwrap(), Command::Decode(decode::DecodeArgs { input, output }) => { let input_bytes = get_input_bytes(input.as_ref()).unwrap(); @@ -77,7 +51,7 @@ impl Command { let value = crate::dynamic_schema::deserialize_from_schema(&mut buf, &schema) .expect("Unable to deserialize according to embedded schema"); - output_json(output.as_ref(), &value); + output_json(output.as_ref(), &value).unwrap(); } Command::Extract(extract::ExtractArgs { input, output }) => { let input_bytes = get_input_bytes(input.as_ref()).unwrap(); @@ -121,6 +95,8 @@ pub enum IOError { WriteBytes, #[error("Failed to deserialize input as Borsh {0}")] DeserializeBorsh(&'static str), + #[error("Failed to deserialize input as Json")] + DeserializeJson, #[error("Unexpected schema header: {0}")] IncorrectBorshSchemaHeader(String), } @@ -142,7 +118,7 @@ fn read_stdin() -> Result, IOError> { fn output_writer(output: Option<&PathBuf>) -> Result, IOError> { if let Some(o) = output { let f = - fs::File::create(o).map_err(|e| IOError::CreateOutputFile(o.display().to_string()))?; + fs::File::create(o).map_err(|_e| IOError::CreateOutputFile(o.display().to_string()))?; Ok(Box::new(f) as Box) } else { Ok(Box::new(io::stdout()) as Box) From 448ee9c1ea1a317887831eef4bae8dfa497db86a Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 15 Dec 2022 14:17:58 +0900 Subject: [PATCH 07/15] decode file --- src/command/decode.rs | 47 ++++++++++++++++++++++++++++++++++++++++--- src/command/mod.rs | 31 +++++++++------------------- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/src/command/decode.rs b/src/command/decode.rs index 36ad3f7..42ad62d 100644 --- a/src/command/decode.rs +++ b/src/command/decode.rs @@ -1,15 +1,56 @@ -use std::path::PathBuf; +use std::{io::Write, path::PathBuf}; +use borsh::{schema::BorshSchemaContainer, BorshDeserialize}; use clap::Args; +use super::{get_input_bytes, output_json, output_writer, Execute, IOError}; + #[derive(Args, Debug)] /// Decode Borsh input to JSON. /// /// Requires the input to contain the embedded schema. pub struct DecodeArgs { /// Read input from this file, otherwise from stdin. - pub input: Option, + pub input_path: Option, /// Write output to this file, otherwise to stdout. - pub output: Option, + pub output_path: Option, +} + +pub struct Decode { + pub input: serde_json::Value, + pub output: Box, +} + +impl TryFrom<&'_ DecodeArgs> for Decode { + type Error = IOError; + + fn try_from( + DecodeArgs { + input_path, + output_path, + }: &'_ DecodeArgs, + ) -> Result { + Ok(Self { + input: { + let input_bytes = get_input_bytes(input_path.as_ref())?; + + let mut buf = &input_bytes as &[u8]; + + let schema = ::deserialize(&mut buf) + .map_err(|_| IOError::DeserializeBorsh("schema"))?; + + let value = crate::dynamic_schema::deserialize_from_schema(&mut buf, &schema) + .map_err(|_| IOError::DeserializeBorsh("data according to embedded schema"))?; + value + }, + output: output_writer(output_path.as_ref())?, + }) + } +} + +impl Execute for Decode { + fn execute(&mut self) -> Result<(), IOError> { + output_json(&mut self.output, &self.input) + } } diff --git a/src/command/mod.rs b/src/command/mod.rs index e994a7f..c1b82f0 100644 --- a/src/command/mod.rs +++ b/src/command/mod.rs @@ -4,14 +4,12 @@ use std::{ path::PathBuf, }; -use borsh::{schema::BorshSchemaContainer, BorshDeserialize, BorshSchema, BorshSerialize}; -use clap::{Args, Subcommand}; +use borsh::{schema::BorshSchemaContainer, BorshDeserialize, BorshSerialize}; +use clap::Subcommand; use serde::Serialize; use thiserror::Error; -use crate::json_borsh::JsonSerializableAsBorsh; - -use self::{encode::Encode, pack::Pack, unpack::Unpack}; +use self::{decode::Decode, encode::Encode, pack::Pack, unpack::Unpack}; mod decode; mod encode; @@ -40,19 +38,7 @@ impl Command { Command::Pack(args) => Pack::execute(&mut args.try_into().unwrap()).unwrap(), Command::Unpack(args) => Unpack::execute(&mut args.try_into().unwrap()).unwrap(), Command::Encode(args) => Encode::execute(&mut args.try_into().unwrap()).unwrap(), - Command::Decode(decode::DecodeArgs { input, output }) => { - let input_bytes = get_input_bytes(input.as_ref()).unwrap(); - - let mut buf = &input_bytes as &[u8]; - - let schema = - ::deserialize(&mut buf).unwrap(); - - let value = crate::dynamic_schema::deserialize_from_schema(&mut buf, &schema) - .expect("Unable to deserialize according to embedded schema"); - - output_json(output.as_ref(), &value).unwrap(); - } + Command::Decode(args) => Decode::execute(&mut args.try_into().unwrap()).unwrap(), Command::Extract(extract::ExtractArgs { input, output }) => { let input_bytes = get_input_bytes(input.as_ref()).unwrap(); @@ -125,7 +111,7 @@ fn output_writer(output: Option<&PathBuf>) -> Result, IOError> { } } -fn output_bytes(writer: &mut impl Write, value: &[u8]) -> Result<(), IOError> { +fn output_bytes(mut writer: impl Write, value: &[u8]) -> Result<(), IOError> { writer.write_all(value).map_err(|_| IOError::WriteBytes) } @@ -148,8 +134,7 @@ fn output_borsh( .expect("Failed to write Borsh"); } -fn output_json(output: Option<&PathBuf>, value: &impl Serialize) -> Result<(), IOError> { - let writer = output_writer(output)?; +fn output_json(writer: impl Write, value: &impl Serialize) -> Result<(), IOError> { serde_json::to_writer(writer, value).map_err(|_| IOError::WriteJson) } @@ -159,6 +144,8 @@ mod tests { use borsh::{BorshSchema, BorshSerialize}; use serde::Serialize; + use crate::command::output_writer; + use super::{output_borsh, output_json}; #[test] @@ -207,7 +194,7 @@ mod tests { }; borsh::try_to_vec_with_schema(&v); output_json( - Some(&"./dataonly.json".into()), + &mut output_writer(Some(&"./dataonly.json".into())).unwrap(), &v, // Some(&First::schema_container()), ); From 0d771a3b507bf34c625136b634db5e9b8f1d5e2e Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 15 Dec 2022 15:07:32 +0900 Subject: [PATCH 08/15] extract and strip --- src/command/encode.rs | 4 +-- src/command/extract.rs | 34 ++++++++++++++++++++++++- src/command/mod.rs | 56 ++++++++---------------------------------- src/command/pack.rs | 4 +-- src/command/strip.rs | 39 ++++++++++++++++++++++++++++- 5 files changed, 85 insertions(+), 52 deletions(-) diff --git a/src/command/encode.rs b/src/command/encode.rs index d43a65c..25417fd 100644 --- a/src/command/encode.rs +++ b/src/command/encode.rs @@ -5,7 +5,7 @@ use clap::Args; use crate::{dynamic_schema::serialize_with_schema, json_borsh::JsonSerializableAsBorsh}; -use super::{get_input_bytes, output_borsh2, output_writer, Execute, IOError}; +use super::{get_input_bytes, output_borsh, output_writer, Execute, IOError}; #[derive(Args, Debug)] /// Convert JSON to Borsh. @@ -61,7 +61,7 @@ impl Execute for Encode { serialize_with_schema(writer, &self.input, &schema).map_err(|_| IOError::WriteBorsh)?; Ok(()) } else { - output_borsh2(writer, &JsonSerializableAsBorsh(&self.input)) + output_borsh(writer, &JsonSerializableAsBorsh(&self.input)) } } } diff --git a/src/command/extract.rs b/src/command/extract.rs index 51f3638..ab1fc14 100644 --- a/src/command/extract.rs +++ b/src/command/extract.rs @@ -1,7 +1,10 @@ -use std::path::PathBuf; +use std::{io::Write, path::PathBuf}; +use borsh::{schema::BorshSchemaContainer, BorshDeserialize}; use clap::Args; +use super::{get_input_bytes, output_borsh, output_writer, Execute, IOError}; + #[derive(Args, Debug)] /// Extract the Borsh schema header. pub struct ExtractArgs { @@ -11,3 +14,32 @@ pub struct ExtractArgs { /// Write output to this file, otherwise to stdout. pub output: Option, } + +pub struct Extract { + pub schema: BorshSchemaContainer, + pub output: Box, +} + +impl TryFrom<&'_ ExtractArgs> for Extract { + type Error = IOError; + + fn try_from(ExtractArgs { input, output }: &'_ ExtractArgs) -> Result { + let input_bytes = get_input_bytes(input.as_ref())?; + + let mut buf = &input_bytes as &[u8]; + + let schema = ::deserialize(&mut buf) + .map_err(|_| IOError::DeserializeBorsh("schema"))?; + + Ok(Self { + schema, + output: output_writer(output.as_ref())?, + }) + } +} + +impl Execute for Extract { + fn execute(&mut self) -> Result<(), IOError> { + output_borsh(&mut self.output, &self.schema) + } +} diff --git a/src/command/mod.rs b/src/command/mod.rs index c1b82f0..2c5f410 100644 --- a/src/command/mod.rs +++ b/src/command/mod.rs @@ -4,12 +4,14 @@ use std::{ path::PathBuf, }; -use borsh::{schema::BorshSchemaContainer, BorshDeserialize, BorshSerialize}; +use borsh::BorshSerialize; use clap::Subcommand; use serde::Serialize; use thiserror::Error; -use self::{decode::Decode, encode::Encode, pack::Pack, unpack::Unpack}; +use self::{ + decode::Decode, encode::Encode, extract::Extract, pack::Pack, strip::Strip, unpack::Unpack, +}; mod decode; mod encode; @@ -39,28 +41,8 @@ impl Command { Command::Unpack(args) => Unpack::execute(&mut args.try_into().unwrap()).unwrap(), Command::Encode(args) => Encode::execute(&mut args.try_into().unwrap()).unwrap(), Command::Decode(args) => Decode::execute(&mut args.try_into().unwrap()).unwrap(), - Command::Extract(extract::ExtractArgs { input, output }) => { - let input_bytes = get_input_bytes(input.as_ref()).unwrap(); - - let mut buf = &input_bytes as &[u8]; - - let schema = - ::deserialize(&mut buf).unwrap(); - - output_borsh(output.as_ref(), &schema, None); - } - Command::Strip(strip::StripArgs { input, output }) => { - let input_bytes = get_input_bytes(input.as_ref()).unwrap(); - - let mut buf = &input_bytes as &[u8]; - - let _ = ::deserialize(&mut buf).unwrap(); - - output_writer(output.as_ref()) - .unwrap() - .write_all(buf) - .expect("Unable to write output"); - } + Command::Extract(args) => Extract::execute(&mut args.try_into().unwrap()).unwrap(), + Command::Strip(args) => Strip::execute(&mut args.try_into().unwrap()).unwrap(), } } } @@ -115,25 +97,10 @@ fn output_bytes(mut writer: impl Write, value: &[u8]) -> Result<(), IOError> { writer.write_all(value).map_err(|_| IOError::WriteBytes) } -fn output_borsh2(writer: impl Write, value: impl BorshSerialize) -> Result<(), IOError> { +fn output_borsh(writer: impl Write, value: impl BorshSerialize) -> Result<(), IOError> { borsh::to_writer(writer, &value).map_err(|_| IOError::WriteBorsh) } -fn output_borsh( - output: Option<&PathBuf>, - value: &impl BorshSerialize, - schema: Option<&BorshSchemaContainer>, -) { - let writer = output_writer(output).unwrap(); - - if let Some(schema) = schema { - borsh::to_writer(writer, &(schema, value)) - } else { - borsh::to_writer(writer, value) - } - .expect("Failed to write Borsh"); -} - fn output_json(writer: impl Write, value: &impl Serialize) -> Result<(), IOError> { serde_json::to_writer(writer, value).map_err(|_| IOError::WriteJson) } @@ -144,9 +111,7 @@ mod tests { use borsh::{BorshSchema, BorshSerialize}; use serde::Serialize; - use crate::command::output_writer; - - use super::{output_borsh, output_json}; + use crate::command::{output_borsh, output_json, output_writer}; #[test] fn test_schema() { @@ -199,9 +164,8 @@ mod tests { // Some(&First::schema_container()), ); output_borsh( - Some(&"./dataandschema.borsh".into()), - &v, - Some(&First::schema_container()), + &mut output_writer(Some(&"./dataandschema.borsh".into())).unwrap(), + (&v, &First::schema_container()), ); } diff --git a/src/command/pack.rs b/src/command/pack.rs index 3a978aa..002e3cc 100644 --- a/src/command/pack.rs +++ b/src/command/pack.rs @@ -38,10 +38,10 @@ impl TryFrom<&'_ PackArgs> for Pack { impl super::Execute for Pack { fn execute(&mut self) -> Result<(), super::IOError> { if self.no_schema { - super::output_borsh2(&mut self.output, &self.input) + super::output_borsh(&mut self.output, &self.input) } else { let schema = Vec::::schema_container(); - super::output_borsh2(&mut self.output, &(schema, &self.input)) + super::output_borsh(&mut self.output, &(schema, &self.input)) } } } diff --git a/src/command/strip.rs b/src/command/strip.rs index 0337f56..d9db612 100644 --- a/src/command/strip.rs +++ b/src/command/strip.rs @@ -1,7 +1,10 @@ -use std::path::PathBuf; +use std::{io::Write, path::PathBuf}; +use borsh::{schema::BorshSchemaContainer, BorshDeserialize}; use clap::Args; +use super::{get_input_bytes, output_bytes, output_writer, Execute, IOError}; + #[derive(Args, Debug)] /// Remove the Borsh schema header. pub struct StripArgs { @@ -11,3 +14,37 @@ pub struct StripArgs { /// Write output to this file, otherwise to stdout. pub output: Option, } + +pub struct Strip { + pub input_bytes: Vec, + pub schema_length: usize, + pub output: Box, +} + +impl TryFrom<&'_ StripArgs> for Strip { + type Error = IOError; + + fn try_from(StripArgs { input, output }: &'_ StripArgs) -> Result { + let input_bytes = get_input_bytes(input.as_ref())?; + + let mut buf = &input_bytes as &[u8]; + let buf_length = buf.len(); + + let _ = ::deserialize(&mut buf) + .map_err(|_| IOError::DeserializeBorsh("schema")); + + let schema_length = buf_length - buf.len(); + + Ok(Self { + input_bytes, + schema_length, + output: output_writer(output.as_ref())?, + }) + } +} + +impl Execute for Strip { + fn execute(&mut self) -> Result<(), IOError> { + output_bytes(&mut self.output, &self.input_bytes[self.schema_length..]) + } +} From 876cf2657dcf260bc119430b82d56e89a9ddc4a7 Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 20 Dec 2022 18:01:51 +0900 Subject: [PATCH 09/15] chore: pack tests --- src/command/pack.rs | 57 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/src/command/pack.rs b/src/command/pack.rs index 002e3cc..f30b6a4 100644 --- a/src/command/pack.rs +++ b/src/command/pack.rs @@ -17,13 +17,13 @@ pub struct PackArgs { pub no_schema: bool, } -pub struct Pack { +pub struct Pack<'a> { pub input: Vec, - pub output: Box, + pub output: Box, pub no_schema: bool, } -impl TryFrom<&'_ PackArgs> for Pack { +impl TryFrom<&'_ PackArgs> for Pack<'_> { type Error = super::IOError; fn try_from(args: &'_ PackArgs) -> Result { @@ -35,7 +35,7 @@ impl TryFrom<&'_ PackArgs> for Pack { } } -impl super::Execute for Pack { +impl super::Execute for Pack<'_> { fn execute(&mut self) -> Result<(), super::IOError> { if self.no_schema { super::output_borsh(&mut self.output, &self.input) @@ -45,3 +45,52 @@ impl super::Execute for Pack { } } } + +#[cfg(test)] +mod tests { + use std::io::BufWriter; + + use crate::command::Execute; + + use super::Pack; + + #[test] + fn with_schema() { + let test_vector = vec![1u8, 2, 3, 4]; + let mut output_vector: Vec = vec![]; + let writer = BufWriter::new(&mut output_vector); + + let mut p = Pack { + input: test_vector.clone(), + output: Box::new(writer), + no_schema: false, + }; + + p.execute().unwrap(); + drop(p); + + let expected = borsh::try_to_vec_with_schema(&test_vector).unwrap(); + + assert_eq!(expected, output_vector); + } + + #[test] + fn without_schema() { + let test_vector = vec![1u8, 2, 3, 4]; + let mut output_vector: Vec = vec![]; + let writer = BufWriter::new(&mut output_vector); + + let mut p = Pack { + input: test_vector.clone(), + output: Box::new(writer), + no_schema: true, + }; + + p.execute().unwrap(); + drop(p); + + let expected = borsh::to_vec(&test_vector).unwrap(); + + assert_eq!(expected, output_vector); + } +} From 4bf12aaa55be016a358472d6b24cd360084434df Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 20 Dec 2022 18:17:10 +0900 Subject: [PATCH 10/15] chore: unpack tests --- src/command/unpack.rs | 61 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/src/command/unpack.rs b/src/command/unpack.rs index c4f9244..e4b43c3 100644 --- a/src/command/unpack.rs +++ b/src/command/unpack.rs @@ -19,7 +19,13 @@ pub struct UnpackArgs { pub no_schema: bool, } -impl TryFrom<&'_ UnpackArgs> for Unpack { +pub struct Unpack<'a> { + pub input: Vec, + pub output: Box, + pub no_schema: bool, +} + +impl TryFrom<&'_ UnpackArgs> for Unpack<'_> { type Error = super::IOError; fn try_from(args: &'_ UnpackArgs) -> Result { Ok(Self { @@ -30,13 +36,7 @@ impl TryFrom<&'_ UnpackArgs> for Unpack { } } -pub struct Unpack { - pub input: Vec, - pub output: Box, - pub no_schema: bool, -} - -impl super::Execute for Unpack { +impl super::Execute for Unpack<'_> { fn execute(&mut self) -> Result<(), super::IOError> { let value = if self.no_schema { Vec::::try_from_slice(&self.input) @@ -53,3 +53,48 @@ impl super::Execute for Unpack { output_bytes(&mut self.output, &value) } } + +#[cfg(test)] +mod tests { + use std::io::BufWriter; + + use crate::command::Execute; + + use super::Unpack; + + #[test] + fn with_schema() { + let test_vector = vec![1u8, 2, 3, 4]; + let mut output_vector: Vec = vec![]; + let writer = BufWriter::new(&mut output_vector); + + let mut p = Unpack { + input: borsh::try_to_vec_with_schema(&test_vector).unwrap(), + output: Box::new(writer), + no_schema: false, + }; + + p.execute().unwrap(); + drop(p); + + assert_eq!(test_vector, output_vector); + } + + #[test] + fn without_schema() { + let test_vector = vec![1u8, 2, 3, 4]; + let mut output_vector: Vec = vec![]; + let writer = BufWriter::new(&mut output_vector); + + let mut p = Unpack { + input: borsh::to_vec(&test_vector).unwrap().clone(), + output: Box::new(writer), + no_schema: true, + }; + + p.execute().unwrap(); + drop(p); + + assert_eq!(test_vector, output_vector); + } +} From 8ed0fe1a3663c307dfdc37b5ba346ada31d7063b Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 20 Dec 2022 18:43:54 +0900 Subject: [PATCH 11/15] chore: encode tests --- src/command/encode.rs | 105 ++++++++++++++++++++++++++++++++++++++++-- src/command/mod.rs | 1 + 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/src/command/encode.rs b/src/command/encode.rs index 25417fd..0a4313b 100644 --- a/src/command/encode.rs +++ b/src/command/encode.rs @@ -24,13 +24,13 @@ pub struct EncodeArgs { pub schema: Option, } -pub struct Encode { +pub struct Encode<'a> { pub input: serde_json::Value, - pub output: Box, + pub output: Box, pub schema: Option, } -impl TryFrom<&'_ EncodeArgs> for Encode { +impl TryFrom<&'_ EncodeArgs> for Encode<'_> { type Error = IOError; fn try_from(args: &'_ EncodeArgs) -> Result { @@ -53,7 +53,7 @@ impl TryFrom<&'_ EncodeArgs> for Encode { } } -impl Execute for Encode { +impl Execute for Encode<'_> { fn execute(&mut self) -> Result<(), IOError> { let writer = &mut self.output; if let Some(schema) = &self.schema { @@ -65,3 +65,100 @@ impl Execute for Encode { } } } + +#[cfg(test)] +mod tests { + use std::io::BufWriter; + + use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; + use serde::{Deserialize, Serialize}; + + use crate::command::Execute; + + use super::Encode; + + #[derive(Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, Debug)] + struct Parent { + integer: u32, + vector: [u8; 8], + child: Child, + } + + #[derive(Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, PartialEq, Debug)] + struct JsonParent { + integer: f64, + vector: Vec, + child: Child, + } + + #[derive(Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, PartialEq, Debug)] + struct Child { + string: String, + boolean: bool, + } + + #[test] + fn with_schema() { + let value = Parent { + integer: 24, + vector: [8, 7, 6, 5, 4, 3, 2, 1], + child: Child { + string: "()".to_string(), + boolean: false, + }, + }; + + let mut output_vector: Vec = vec![]; + let writer = BufWriter::new(&mut output_vector); + + let mut p = Encode { + input: serde_json::to_value(&value).unwrap(), + output: Box::new(writer), + schema: Some(Parent::schema_container()), + }; + + p.execute().unwrap(); + drop(p); + + let expected = borsh::try_to_vec_with_schema(&value).unwrap(); + + assert_eq!(expected, output_vector); + } + + #[test] + fn without_schema() { + let value = Parent { + integer: 24, + vector: [8, 7, 6, 5, 4, 3, 2, 1], + child: Child { + string: "()".to_string(), + boolean: false, + }, + }; + let expected = JsonParent { + integer: 24.0, + vector: vec![8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0], + child: Child { + string: "()".to_string(), + boolean: false, + }, + }; + + let mut output_vector: Vec = vec![]; + let writer = BufWriter::new(&mut output_vector); + + let mut p = Encode { + input: serde_json::to_value(&value).unwrap(), + output: Box::new(writer), + schema: None, + }; + + p.execute().unwrap(); + drop(p); + + assert_eq!( + ::try_from_slice(&output_vector).unwrap(), + expected, + ); + } +} diff --git a/src/command/mod.rs b/src/command/mod.rs index 2c5f410..8cd23b6 100644 --- a/src/command/mod.rs +++ b/src/command/mod.rs @@ -114,6 +114,7 @@ mod tests { use crate::command::{output_borsh, output_json, output_writer}; #[test] + #[ignore = "pollution"] fn test_schema() { #[derive(BorshSerialize, BorshSchema, Serialize)] struct First { From ee3c3885d29dde782a2ea718990a8d3bbc65cb3d Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 20 Dec 2022 18:56:23 +0900 Subject: [PATCH 12/15] chore: decode tests --- src/command/decode.rs | 90 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/src/command/decode.rs b/src/command/decode.rs index 42ad62d..1a601de 100644 --- a/src/command/decode.rs +++ b/src/command/decode.rs @@ -17,12 +17,12 @@ pub struct DecodeArgs { pub output_path: Option, } -pub struct Decode { - pub input: serde_json::Value, - pub output: Box, +pub struct Decode<'a> { + pub input: Vec, + pub output: Box, } -impl TryFrom<&'_ DecodeArgs> for Decode { +impl TryFrom<&'_ DecodeArgs> for Decode<'_> { type Error = IOError; fn try_from( @@ -32,25 +32,79 @@ impl TryFrom<&'_ DecodeArgs> for Decode { }: &'_ DecodeArgs, ) -> Result { Ok(Self { - input: { - let input_bytes = get_input_bytes(input_path.as_ref())?; - - let mut buf = &input_bytes as &[u8]; - - let schema = ::deserialize(&mut buf) - .map_err(|_| IOError::DeserializeBorsh("schema"))?; - - let value = crate::dynamic_schema::deserialize_from_schema(&mut buf, &schema) - .map_err(|_| IOError::DeserializeBorsh("data according to embedded schema"))?; - value - }, + input: get_input_bytes(input_path.as_ref())?, output: output_writer(output_path.as_ref())?, }) } } -impl Execute for Decode { +impl Execute for Decode<'_> { fn execute(&mut self) -> Result<(), IOError> { - output_json(&mut self.output, &self.input) + let mut buf = &self.input as &[u8]; + + let schema = ::deserialize(&mut buf) + .map_err(|_| IOError::DeserializeBorsh("schema"))?; + + let value = crate::dynamic_schema::deserialize_from_schema(&mut buf, &schema) + .map_err(|_| IOError::DeserializeBorsh("data according to embedded schema"))?; + + output_json(&mut self.output, &value) + } +} + +#[cfg(test)] +mod tests { + use std::io::BufWriter; + + use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; + use serde::{Deserialize, Serialize}; + + use crate::command::Execute; + + use super::Decode; + + #[derive(Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, Debug)] + struct Parent { + integer: u32, + vector: [u8; 8], + child: Child, + } + + #[derive( + Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, PartialEq, Debug, + )] + struct Child { + string: String, + boolean: bool, + } + + #[test] + fn with_schema() { + let value = Parent { + integer: 24, + vector: [8, 7, 6, 5, 4, 3, 2, 1], + child: Child { + string: "()".to_string(), + boolean: false, + }, + }; + + let mut output_vector: Vec = vec![]; + let writer = BufWriter::new(&mut output_vector); + + let mut p = Decode { + input: borsh::try_to_vec_with_schema(&value).unwrap(), + output: Box::new(writer), + }; + + p.execute().unwrap(); + drop(p); + + let expected = serde_json::to_value(&value).unwrap(); + + assert_eq!( + expected, + serde_json::from_slice::(&output_vector).unwrap() + ); } } From 3fa856c3cc2a2c3c10a02628609af68d15cc4039 Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 20 Dec 2022 19:06:29 +0900 Subject: [PATCH 13/15] chore: extract and strip tests --- src/command/decode.rs | 2 +- src/command/encode.rs | 2 +- src/command/extract.rs | 81 +++++++++++++++++++++++++++++++++------- src/command/strip.rs | 85 ++++++++++++++++++++++++++++++++---------- 4 files changed, 136 insertions(+), 34 deletions(-) diff --git a/src/command/decode.rs b/src/command/decode.rs index 1a601de..353d26e 100644 --- a/src/command/decode.rs +++ b/src/command/decode.rs @@ -79,7 +79,7 @@ mod tests { } #[test] - fn with_schema() { + fn test() { let value = Parent { integer: 24, vector: [8, 7, 6, 5, 4, 3, 2, 1], diff --git a/src/command/encode.rs b/src/command/encode.rs index 0a4313b..762b4e3 100644 --- a/src/command/encode.rs +++ b/src/command/encode.rs @@ -58,7 +58,7 @@ impl Execute for Encode<'_> { let writer = &mut self.output; if let Some(schema) = &self.schema { BorshSerialize::serialize(&schema, writer).map_err(|_| IOError::WriteBorsh)?; - serialize_with_schema(writer, &self.input, &schema).map_err(|_| IOError::WriteBorsh)?; + serialize_with_schema(writer, &self.input, schema).map_err(|_| IOError::WriteBorsh)?; Ok(()) } else { output_borsh(writer, &JsonSerializableAsBorsh(&self.input)) diff --git a/src/command/extract.rs b/src/command/extract.rs index ab1fc14..ce4ce91 100644 --- a/src/command/extract.rs +++ b/src/command/extract.rs @@ -15,31 +15,86 @@ pub struct ExtractArgs { pub output: Option, } -pub struct Extract { - pub schema: BorshSchemaContainer, - pub output: Box, +pub struct Extract<'a> { + pub input: Vec, + pub output: Box, } -impl TryFrom<&'_ ExtractArgs> for Extract { +impl TryFrom<&'_ ExtractArgs> for Extract<'_> { type Error = IOError; fn try_from(ExtractArgs { input, output }: &'_ ExtractArgs) -> Result { - let input_bytes = get_input_bytes(input.as_ref())?; + Ok(Self { + input: get_input_bytes(input.as_ref())?, + output: output_writer(output.as_ref())?, + }) + } +} - let mut buf = &input_bytes as &[u8]; +impl Execute for Extract<'_> { + fn execute(&mut self) -> Result<(), IOError> { + let mut buf = &self.input as &[u8]; let schema = ::deserialize(&mut buf) .map_err(|_| IOError::DeserializeBorsh("schema"))?; - Ok(Self { - schema, - output: output_writer(output.as_ref())?, - }) + output_borsh(&mut self.output, &schema) } } -impl Execute for Extract { - fn execute(&mut self) -> Result<(), IOError> { - output_borsh(&mut self.output, &self.schema) +#[cfg(test)] +mod tests { + use std::io::BufWriter; + + use borsh::{BorshDeserialize, BorshSchema, BorshSerialize, schema::BorshSchemaContainer}; + use serde::{Deserialize, Serialize}; + + use crate::command::Execute; + + use super::Extract; + + #[derive(Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, Debug)] + struct Parent { + integer: u32, + vector: [u8; 8], + child: Child, + } + + #[derive( + Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, PartialEq, Debug, + )] + struct Child { + string: String, + boolean: bool, + } + + #[test] + fn test() { + let value = Parent { + integer: 24, + vector: [8, 7, 6, 5, 4, 3, 2, 1], + child: Child { + string: "()".to_string(), + boolean: false, + }, + }; + + let mut output_vector: Vec = vec![]; + let writer = BufWriter::new(&mut output_vector); + + let mut p = Extract { + input: borsh::try_to_vec_with_schema(&value).unwrap(), + output: Box::new(writer), + }; + + p.execute().unwrap(); + drop(p); + + let expected = Parent::schema_container(); + + assert_eq!( + expected, + BorshSchemaContainer::try_from_slice(&output_vector).unwrap(), + ); } } diff --git a/src/command/strip.rs b/src/command/strip.rs index d9db612..3925764 100644 --- a/src/command/strip.rs +++ b/src/command/strip.rs @@ -15,36 +15,83 @@ pub struct StripArgs { pub output: Option, } -pub struct Strip { - pub input_bytes: Vec, - pub schema_length: usize, - pub output: Box, +pub struct Strip<'a> { + pub input: Vec, + pub output: Box, } -impl TryFrom<&'_ StripArgs> for Strip { +impl TryFrom<&'_ StripArgs> for Strip<'_> { type Error = IOError; fn try_from(StripArgs { input, output }: &'_ StripArgs) -> Result { - let input_bytes = get_input_bytes(input.as_ref())?; - - let mut buf = &input_bytes as &[u8]; - let buf_length = buf.len(); - - let _ = ::deserialize(&mut buf) - .map_err(|_| IOError::DeserializeBorsh("schema")); - - let schema_length = buf_length - buf.len(); - Ok(Self { - input_bytes, - schema_length, + input: get_input_bytes(input.as_ref())?, output: output_writer(output.as_ref())?, }) } } -impl Execute for Strip { +impl Execute for Strip<'_> { fn execute(&mut self) -> Result<(), IOError> { - output_bytes(&mut self.output, &self.input_bytes[self.schema_length..]) + let mut buf = &self.input as &[u8]; + + ::deserialize(&mut buf) + .map_err(|_| IOError::DeserializeBorsh("schema"))?; + + output_bytes(&mut self.output, buf) + } +} + +#[cfg(test)] +mod tests { + use std::io::BufWriter; + + use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; + use serde::{Deserialize, Serialize}; + + use crate::command::Execute; + + use super::Strip; + + #[derive( + Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, Debug, PartialEq, + )] + struct Parent { + integer: u32, + vector: [u8; 8], + child: Child, + } + + #[derive( + Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, PartialEq, Debug, + )] + struct Child { + string: String, + boolean: bool, + } + + #[test] + fn test() { + let value = Parent { + integer: 24, + vector: [8, 7, 6, 5, 4, 3, 2, 1], + child: Child { + string: "()".to_string(), + boolean: false, + }, + }; + + let mut output_vector: Vec = vec![]; + let writer = BufWriter::new(&mut output_vector); + + let mut p = Strip { + input: borsh::try_to_vec_with_schema(&value).unwrap(), + output: Box::new(writer), + }; + + p.execute().unwrap(); + drop(p); + + assert_eq!(value, Parent::try_from_slice(&output_vector).unwrap(),); } } From 37db793f5143a7d58ef79e1c52203d88e7d2345f Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 20 Dec 2022 19:07:41 +0900 Subject: [PATCH 14/15] chore: fmt --- src/command/encode.rs | 8 ++++++-- src/command/extract.rs | 2 +- src/command/unpack.rs | 2 +- src/dynamic_schema.rs | 43 ++++++++++++++++++++---------------------- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/command/encode.rs b/src/command/encode.rs index 762b4e3..b3e4bd7 100644 --- a/src/command/encode.rs +++ b/src/command/encode.rs @@ -84,14 +84,18 @@ mod tests { child: Child, } - #[derive(Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, PartialEq, Debug)] + #[derive( + Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, PartialEq, Debug, + )] struct JsonParent { integer: f64, vector: Vec, child: Child, } - #[derive(Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, PartialEq, Debug)] + #[derive( + Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, PartialEq, Debug, + )] struct Child { string: String, boolean: bool, diff --git a/src/command/extract.rs b/src/command/extract.rs index ce4ce91..0d4ae2c 100644 --- a/src/command/extract.rs +++ b/src/command/extract.rs @@ -46,7 +46,7 @@ impl Execute for Extract<'_> { mod tests { use std::io::BufWriter; - use borsh::{BorshDeserialize, BorshSchema, BorshSerialize, schema::BorshSchemaContainer}; + use borsh::{schema::BorshSchemaContainer, BorshDeserialize, BorshSchema, BorshSerialize}; use serde::{Deserialize, Serialize}; use crate::command::Execute; diff --git a/src/command/unpack.rs b/src/command/unpack.rs index e4b43c3..55061de 100644 --- a/src/command/unpack.rs +++ b/src/command/unpack.rs @@ -87,7 +87,7 @@ mod tests { let writer = BufWriter::new(&mut output_vector); let mut p = Unpack { - input: borsh::to_vec(&test_vector).unwrap().clone(), + input: borsh::to_vec(&test_vector).unwrap(), output: Box::new(writer), no_schema: true, }; diff --git a/src/dynamic_schema.rs b/src/dynamic_schema.rs index cfc2aa7..e03cae5 100644 --- a/src/dynamic_schema.rs +++ b/src/dynamic_schema.rs @@ -80,32 +80,29 @@ fn deserialize_declaration_from_schema( deserialize_declaration_from_schema(buf, schema, variant_declaration) .map(|v| json!({ variant_name: v })) } - Definition::Struct { fields } => { - match fields { - Fields::NamedFields(fields) => { - let mut object = HashMap::::new(); - for &(ref key, ref value_declaration) in fields { - let value = deserialize_declaration_from_schema( - buf, - schema, - value_declaration, - )?; - object.insert(key.to_string(), value); - } - Ok(serde_json::to_value(object)?) + Definition::Struct { fields } => match fields { + Fields::NamedFields(fields) => { + let mut object = HashMap::::new(); + for &(ref key, ref value_declaration) in fields { + let value = deserialize_declaration_from_schema( + buf, + schema, + value_declaration, + )?; + object.insert(key.to_string(), value); } - Fields::UnnamedFields(elements) => { - let mut v = Vec::::with_capacity(elements.len()); - for element in elements { - let e = - deserialize_declaration_from_schema(buf, schema, element)?; - v.push(e); - } - Ok(v.into()) + Ok(serde_json::to_value(object)?) + } + Fields::UnnamedFields(elements) => { + let mut v = Vec::::with_capacity(elements.len()); + for element in elements { + let e = deserialize_declaration_from_schema(buf, schema, element)?; + v.push(e); } - Fields::Empty => Ok(Vec::::new().into()), + Ok(v.into()) } - } + Fields::Empty => Ok(Vec::::new().into()), + }, } } else { todo!("Unknown type to deserialize: {declaration}") From 1f72feedc63365694376ebb4fe305c7d7eb3a31d Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 20 Dec 2022 19:53:07 +0900 Subject: [PATCH 15/15] v0.1.3 --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/command/decode.rs | 10 +++++++++- src/command/mod.rs | 32 ++++++++++++++++++++++---------- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b28e1a..2e139c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,7 +43,7 @@ dependencies = [ [[package]] name = "borsh-cli" -version = "0.1.2" +version = "0.1.3" dependencies = [ "anyhow", "borsh", diff --git a/Cargo.toml b/Cargo.toml index 292fa53..cf1c35d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ description = "Command-line utility for manipulating Borsh-serialized data" edition = "2021" license = "GPL-3.0" name = "borsh-cli" -version = "0.1.2" +version = "0.1.3" [dependencies] anyhow = "1.0.66" diff --git a/src/command/decode.rs b/src/command/decode.rs index 353d26e..da7ac0f 100644 --- a/src/command/decode.rs +++ b/src/command/decode.rs @@ -15,11 +15,16 @@ pub struct DecodeArgs { /// Write output to this file, otherwise to stdout. pub output_path: Option, + + /// Format output + #[arg(short, long)] + pub pretty: bool, } pub struct Decode<'a> { pub input: Vec, pub output: Box, + pub pretty: bool, } impl TryFrom<&'_ DecodeArgs> for Decode<'_> { @@ -29,11 +34,13 @@ impl TryFrom<&'_ DecodeArgs> for Decode<'_> { DecodeArgs { input_path, output_path, + pretty, }: &'_ DecodeArgs, ) -> Result { Ok(Self { input: get_input_bytes(input_path.as_ref())?, output: output_writer(output_path.as_ref())?, + pretty: *pretty, }) } } @@ -48,7 +55,7 @@ impl Execute for Decode<'_> { let value = crate::dynamic_schema::deserialize_from_schema(&mut buf, &schema) .map_err(|_| IOError::DeserializeBorsh("data according to embedded schema"))?; - output_json(&mut self.output, &value) + output_json(&mut self.output, &value, self.pretty) } } @@ -95,6 +102,7 @@ mod tests { let mut p = Decode { input: borsh::try_to_vec_with_schema(&value).unwrap(), output: Box::new(writer), + pretty: false, }; p.execute().unwrap(); diff --git a/src/command/mod.rs b/src/command/mod.rs index 8cd23b6..485fe7f 100644 --- a/src/command/mod.rs +++ b/src/command/mod.rs @@ -36,13 +36,20 @@ pub enum Command { impl Command { pub fn run(&self) { - match self { - Command::Pack(args) => Pack::execute(&mut args.try_into().unwrap()).unwrap(), - Command::Unpack(args) => Unpack::execute(&mut args.try_into().unwrap()).unwrap(), - Command::Encode(args) => Encode::execute(&mut args.try_into().unwrap()).unwrap(), - Command::Decode(args) => Decode::execute(&mut args.try_into().unwrap()).unwrap(), - Command::Extract(args) => Extract::execute(&mut args.try_into().unwrap()).unwrap(), - Command::Strip(args) => Strip::execute(&mut args.try_into().unwrap()).unwrap(), + #[inline] + fn run_args(args: impl TryInto) -> Result<(), IOError> { + E::execute(&mut args.try_into()?) + } + + if let Err(e) = match self { + Command::Pack(args) => run_args::(args), + Command::Unpack(args) => run_args::(args), + Command::Encode(args) => run_args::(args), + Command::Decode(args) => run_args::(args), + Command::Extract(args) => run_args::(args), + Command::Strip(args) => run_args::(args), + } { + eprintln!("Error: {e}"); } } } @@ -63,7 +70,7 @@ pub enum IOError { WriteBytes, #[error("Failed to deserialize input as Borsh {0}")] DeserializeBorsh(&'static str), - #[error("Failed to deserialize input as Json")] + #[error("Failed to deserialize input as JSON")] DeserializeJson, #[error("Unexpected schema header: {0}")] IncorrectBorshSchemaHeader(String), @@ -101,8 +108,12 @@ fn output_borsh(writer: impl Write, value: impl BorshSerialize) -> Result<(), IO borsh::to_writer(writer, &value).map_err(|_| IOError::WriteBorsh) } -fn output_json(writer: impl Write, value: &impl Serialize) -> Result<(), IOError> { - serde_json::to_writer(writer, value).map_err(|_| IOError::WriteJson) +fn output_json(writer: impl Write, value: &impl Serialize, pretty: bool) -> Result<(), IOError> { + if pretty { + serde_json::to_writer_pretty(writer, value).map_err(|_| IOError::WriteJson) + } else { + serde_json::to_writer(writer, value).map_err(|_| IOError::WriteJson) + } } #[cfg(test)] @@ -162,6 +173,7 @@ mod tests { output_json( &mut output_writer(Some(&"./dataonly.json".into())).unwrap(), &v, + false, // Some(&First::schema_container()), ); output_borsh(