From 60d294c6c8bc1c4fe7cab1171de543d9c991a9f3 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 20 Dec 2022 13:08:33 +0100 Subject: [PATCH] Conversion binary. --- src/bin/cli.rs | 3 ++ src/bin/convert.rs | 61 ++++++++++++++++++++++++ src/graph/flow.rs | 2 +- src/io.rs | 98 ++++++++++++++++++++++++++++++++++++++- src/safe_db/db.rs | 8 +++- src/safe_db/mod.rs | 4 +- src/safe_db/safes_json.rs | 11 +---- src/types/address.rs | 4 ++ src/types/edge.rs | 4 ++ src/types/token.rs | 2 + src/types/u256.rs | 34 ++++++++++++++ tests/integration.rs | 12 ++--- 12 files changed, 221 insertions(+), 22 deletions(-) create mode 100644 src/bin/convert.rs diff --git a/src/bin/cli.rs b/src/bin/cli.rs index d18cdcb..b8af38f 100644 --- a/src/bin/cli.rs +++ b/src/bin/cli.rs @@ -7,8 +7,11 @@ use pathfinder2::io; use pathfinder2::types::Address; use pathfinder2::types::U256; +#[allow(dead_code)] const HUB_ADDRESS: &str = "0x29b9a7fBb8995b2423a71cC17cf9810798F6C543"; +#[allow(dead_code)] const TRANSFER_THROUGH_SIG: &str = "transferThrough(address[],address[],address[],uint256[])"; +#[allow(dead_code)] const RPC_URL: &str = "https://rpc.gnosischain.com"; fn main() { diff --git a/src/bin/convert.rs b/src/bin/convert.rs new file mode 100644 index 0000000..d9c3afc --- /dev/null +++ b/src/bin/convert.rs @@ -0,0 +1,61 @@ +use std::env; + +use pathfinder2::{ + io::{read_edges_binary, read_edges_csv, write_edges_binary, write_edges_csv}, + safe_db::safes_json::import_from_safes_json, +}; + +fn main() { + let operation = env::args().nth(1).and_then(|op| { + if matches!( + op.as_str(), + "--safes-json-to-edges-bin" + | "--safes-json-to-edges-csv" + | "--edges-csv-to-edges-bin" + | "--edges-bin-to-edges-csv" + ) { + Some(op) + } else { + None + } + }); + if env::args().len() != 4 || operation.is_none() { + println!("Usage: convert --safes-json-to-edges-bin "); + println!("Usage: convert --safes-json-to-edges-csv "); + println!("Usage: convert --edges-csv-to-edges-bin "); + println!("Usage: convert --edges-bin-to-edges-csv "); + return; + } + + let input = env::args().nth(2).unwrap(); + let output = env::args().nth(3).unwrap(); + match operation.unwrap().as_str() { + "--safes-json-to-edges-bin" => { + let safes = import_from_safes_json(&input); + let edges = safes.edges(); + println!("Imported {} edges.", edges.edge_count()); + write_edges_binary(edges, &output).unwrap(); + println!("Export done."); + } + "--safes-json-to-edges-csv" => { + let safes = import_from_safes_json(&input); + let edges = safes.edges(); + println!("Imported {} edges.", edges.edge_count()); + write_edges_csv(edges, &output).unwrap(); + println!("Export done."); + } + "--edges-csv-to-edges-bin" => { + let edges = &read_edges_csv(&input).unwrap(); + println!("Imported {} edges.", edges.edge_count()); + write_edges_binary(edges, &output).unwrap(); + println!("Export done."); + } + "--edges-bin-to-edges-csv" => { + let edges = &read_edges_binary(&input).unwrap(); + println!("Imported {} edges.", edges.edge_count()); + write_edges_csv(edges, &output).unwrap(); + println!("Export done."); + } + _ => unreachable!(), + } +} diff --git a/src/graph/flow.rs b/src/graph/flow.rs index 11fae9c..60ae58e 100644 --- a/src/graph/flow.rs +++ b/src/graph/flow.rs @@ -627,7 +627,7 @@ mod test { #[test] fn trust_transfer_limit() { - let (a, b, c, d, t1, t2) = addresses(); + let (a, b, c, d, ..) = addresses(); let edges = build_edges(vec![ // The following two edges should be balance-limited, // i.e. a -> first intermediate is limited by the max of the two. diff --git a/src/io.rs b/src/io.rs index a7ca8fe..4d58aca 100644 --- a/src/io.rs +++ b/src/io.rs @@ -1,6 +1,7 @@ +use std::collections::BTreeSet; use std::fs::File; -use std::io::Read; use std::io::{self, BufRead}; +use std::io::{Read, Write}; use std::{collections::HashMap, io::BufReader}; use crate::types::edge::EdgeDB; @@ -42,6 +43,28 @@ pub fn read_edges_csv(path: &String) -> Result { Ok(EdgeDB::new(edges)) } +pub fn write_edges_binary(edges: &EdgeDB, path: &String) -> Result<(), io::Error> { + let mut file = File::create(path)?; + let address_index = write_address_index(&mut file, edges)?; + write_edges(&mut file, edges, &address_index) +} + +pub fn write_edges_csv(edges: &EdgeDB, path: &String) -> Result<(), io::Error> { + let mut file = File::create(path)?; + let mut sorted_edges = edges.edges().clone(); + sorted_edges.sort(); + for Edge { + from, + to, + token, + capacity, + } in sorted_edges + { + writeln!(file, "{from},{to},{token},{capacity}")?; + } + Ok(()) +} + fn read_address_index(file: &mut File) -> Result, io::Error> { let address_count = read_u32(file)?; let mut addresses = HashMap::new(); @@ -53,18 +76,50 @@ fn read_address_index(file: &mut File) -> Result, io::Erro Ok(addresses) } +fn write_address_index( + file: &mut File, + edges: &EdgeDB, +) -> Result, io::Error> { + let mut addresses = BTreeSet::new(); + for Edge { + from, to, token, .. + } in edges.edges() + { + addresses.insert(*from); + addresses.insert(*to); + addresses.insert(*token); + } + write_u32(file, addresses.len() as u32)?; + let mut index = HashMap::new(); + for (i, addr) in addresses.into_iter().enumerate() { + file.write_all(&addr.to_bytes())?; + index.insert(addr, i as u32); + } + Ok(index) +} + fn read_u32(file: &mut File) -> Result { let mut buf = [0; 4]; file.read_exact(&mut buf)?; Ok(u32::from_be_bytes(buf)) } +fn write_u32(file: &mut File, v: u32) -> Result<(), io::Error> { + let buf = v.to_be_bytes(); + file.write_all(&buf) +} + fn read_u8(file: &mut File) -> Result { let mut buf = [0; 1]; file.read_exact(&mut buf)?; Ok(u8::from_be_bytes(buf)) } +fn write_u8(file: &mut File, v: u8) -> Result<(), io::Error> { + let buf = v.to_be_bytes(); + file.write_all(&buf) +} + fn read_address( file: &mut File, address_index: &HashMap, @@ -73,6 +128,14 @@ fn read_address( Ok(address_index[&index]) } +fn write_address( + file: &mut File, + address: &Address, + address_index: &HashMap, +) -> Result<(), io::Error> { + write_u32(file, *address_index.get(address).unwrap()) +} + fn read_u256(file: &mut File) -> Result { let length = read_u8(file)? as usize; let mut bytes = [0u8; 32]; @@ -82,6 +145,16 @@ fn read_u256(file: &mut File) -> Result { Ok(U256::new(high, low)) } +fn write_u256(file: &mut File, v: &U256) -> Result<(), io::Error> { + let v_bytes = v.to_bytes(); + if v_bytes.is_empty() { + file.write_all(&[1, 0]) + } else { + write_u8(file, v_bytes.len() as u8)?; + file.write_all(&v_bytes) + } +} + fn read_edges(file: &mut File, address_index: &HashMap) -> Result { let edge_count = read_u32(file)?; let mut edges = Vec::new(); @@ -100,6 +173,29 @@ fn read_edges(file: &mut File, address_index: &HashMap) -> Result< Ok(EdgeDB::new(edges)) } +fn write_edges( + file: &mut File, + edges: &EdgeDB, + address_index: &HashMap, +) -> Result<(), io::Error> { + write_u32(file, edges.edge_count() as u32)?; + let mut sorted_edges = edges.edges().clone(); + sorted_edges.sort(); + for Edge { + from, + to, + token, + capacity, + } in &sorted_edges + { + write_address(file, from, address_index)?; + write_address(file, to, address_index)?; + write_address(file, token, address_index)?; + write_u256(file, capacity)?; + } + Ok(()) +} + fn unescape(input: &str) -> &str { match input.chars().next() { Some('"') | Some('\'') => { diff --git a/src/safe_db/db.rs b/src/safe_db/db.rs index f36b62f..942b11a 100644 --- a/src/safe_db/db.rs +++ b/src/safe_db/db.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, default}; +use std::collections::BTreeMap; use crate::types::{edge::EdgeDB, Address, Edge, Safe, U256}; @@ -21,6 +21,10 @@ impl DB { db } + pub fn edges(&self) -> &EdgeDB { + &self.edges + } + fn compute_edges(&mut self) { let mut edges = vec![]; for (user, safe) in &self.safes { @@ -44,7 +48,7 @@ impl DB { // send tokens back to owner for (token, balance) in &safe.balances { if let Some(owner) = self.token_owner.get(token) { - if *user != *owner { + if *user != *owner && *balance != U256::from(0) { edges.push(Edge { from: *user, to: *owner, diff --git a/src/safe_db/mod.rs b/src/safe_db/mod.rs index d50cd0e..dc75007 100644 --- a/src/safe_db/mod.rs +++ b/src/safe_db/mod.rs @@ -1,2 +1,2 @@ -mod db; -mod safes_json; +pub mod db; +pub mod safes_json; diff --git a/src/safe_db/safes_json.rs b/src/safe_db/safes_json.rs index e70dce9..860306f 100644 --- a/src/safe_db/safes_json.rs +++ b/src/safe_db/safes_json.rs @@ -57,6 +57,7 @@ pub fn import_from_safes_json(file: &str) -> DB { #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] struct Safes<'a> { + #[allow(dead_code)] block_number: &'a str, safes: Vec>, } @@ -76,6 +77,7 @@ struct JsonSafe<'a> { #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] struct Edge<'a> { + #[allow(dead_code)] limit: Option<&'a str>, limit_percentage: &'a str, can_send_to_address: &'a str, @@ -104,12 +106,3 @@ struct Token<'a> { struct Owner<'a> { id: &'a str, } - -mod test { - use super::*; - - #[test] - pub fn t1() { - import_from_safes_json("/tmp/safes.json"); - } -} diff --git a/src/types/address.rs b/src/types/address.rs index 891b7b8..9b352a6 100644 --- a/src/types/address.rs +++ b/src/types/address.rs @@ -7,6 +7,10 @@ impl Address { pub fn short(&self) -> String { format!("{self}")[..8].to_string() } + + pub fn to_bytes(self) -> [u8; 20] { + self.0 + } } impl Debug for Address { diff --git a/src/types/edge.rs b/src/types/edge.rs index b2ba1f7..4f0c349 100644 --- a/src/types/edge.rs +++ b/src/types/edge.rs @@ -39,6 +39,10 @@ impl EdgeDB { self.edges.len() } + pub fn edges(&self) -> &Vec { + &self.edges + } + pub fn update(&mut self, update: Edge) { match self.index_of(&update) { Some(i) => self.edges[i].capacity = update.capacity, diff --git a/src/types/token.rs b/src/types/token.rs index 1d74fc6..91cd58a 100644 --- a/src/types/token.rs +++ b/src/types/token.rs @@ -1,6 +1,8 @@ use super::Address; pub struct Token { + #[allow(dead_code)] address: Address, + #[allow(dead_code)] owner: Address, } diff --git a/src/types/u256.rs b/src/types/u256.rs index 64cf818..8f778cc 100644 --- a/src/types/u256.rs +++ b/src/types/u256.rs @@ -29,6 +29,7 @@ impl U256 { let value = BigUint::from(self.0[0]) << 128 | BigUint::from(self.0[1]); format!("{value}") } + pub fn to_decimal_fraction(self) -> String { let value: BigUint = self.into(); let formatted = format!("{value}"); @@ -46,6 +47,19 @@ impl U256 { _ => "0+eps".to_string(), } } + + pub fn to_bytes(&self) -> Vec { + let mut result = Vec::new(); + for i in 0..=1 { + for j in (0..16).rev() { + let b = ((self.0[i] >> (j * 8)) & 0xff) as u8; + if b != 0 || !result.is_empty() { + result.push(b); + } + } + } + result + } } impl From for U256 { @@ -279,4 +293,24 @@ mod test { U256::from("0x8000000000000000000000000000000000000000000000000000000000000000") ); } + + #[test] + fn to_bytes() { + let zero = U256::from("0"); + assert_eq!(zero.to_bytes(), Vec::::new()); + assert_eq!(U256::from("2").to_bytes(), vec![2]); + assert_eq!( + U256::from("0x100000000000000000000000000000000").to_bytes(), + vec![1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + assert_eq!( + U256::from("0xff00000000000000000000000000000001").to_bytes(), + vec![255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + ); + assert_eq!( + U256::from("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + .to_bytes(), + vec![255; 32] + ); + } } diff --git a/tests/integration.rs b/tests/integration.rs index a29f8fd..79af0fb 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1,14 +1,12 @@ use pathfinder2::graph::compute_flow; use pathfinder2::io::read_edges_binary; use pathfinder2::types::edge::EdgeDB; -use pathfinder2::types::{Address, Edge, U256}; -use std::collections::HashMap; +use pathfinder2::types::{Address, U256}; use std::process::Command; -const HUB_ADDRESS: &'static str = "0x29b9a7fBb8995b2423a71cC17cf9810798F6C543"; -const TRANSFER_THROUGH_SIG: &'static str = - "transferThrough(address[],address[],address[],uint256[])"; -const RPC_URL: &'static str = "https://rpc.gnosischain.com"; +const HUB_ADDRESS: &str = "0x29b9a7fBb8995b2423a71cC17cf9810798F6C543"; +const TRANSFER_THROUGH_SIG: &str = "transferThrough(address[],address[],address[],uint256[])"; +const RPC_URL: &str = "https://rpc.gnosischain.com"; #[test] fn test_flow_chris_martin() { @@ -64,7 +62,7 @@ fn test_flow( max_distance: Option, ) { let transfers = compute_flow(source, sink, edges, requested_flow, max_distance); - println!("{:?}", transfers); + println!("{transfers:?}"); let token_owners = transfers .1