Conversion binary.

This commit is contained in:
chriseth 2022-12-20 13:08:33 +01:00
parent 5969204a1f
commit 60d294c6c8
12 changed files with 221 additions and 22 deletions

View file

@ -7,8 +7,11 @@ use pathfinder2::io;
use pathfinder2::types::Address; use pathfinder2::types::Address;
use pathfinder2::types::U256; use pathfinder2::types::U256;
#[allow(dead_code)]
const HUB_ADDRESS: &str = "0x29b9a7fBb8995b2423a71cC17cf9810798F6C543"; const HUB_ADDRESS: &str = "0x29b9a7fBb8995b2423a71cC17cf9810798F6C543";
#[allow(dead_code)]
const TRANSFER_THROUGH_SIG: &str = "transferThrough(address[],address[],address[],uint256[])"; const TRANSFER_THROUGH_SIG: &str = "transferThrough(address[],address[],address[],uint256[])";
#[allow(dead_code)]
const RPC_URL: &str = "https://rpc.gnosischain.com"; const RPC_URL: &str = "https://rpc.gnosischain.com";
fn main() { fn main() {

61
src/bin/convert.rs Normal file
View file

@ -0,0 +1,61 @@
use std::env;
use pathfinder2::{
io::{read_edges_binary, read_edges_csv, write_edges_binary, write_edges_csv},
safe_db::safes_json::import_from_safes_json,
};
fn main() {
let operation = env::args().nth(1).and_then(|op| {
if matches!(
op.as_str(),
"--safes-json-to-edges-bin"
| "--safes-json-to-edges-csv"
| "--edges-csv-to-edges-bin"
| "--edges-bin-to-edges-csv"
) {
Some(op)
} else {
None
}
});
if env::args().len() != 4 || operation.is_none() {
println!("Usage: convert --safes-json-to-edges-bin <safes.json> <edges.dat>");
println!("Usage: convert --safes-json-to-edges-csv <safes.json> <edges.csv>");
println!("Usage: convert --edges-csv-to-edges-bin <edges.csv> <edges.dat>");
println!("Usage: convert --edges-bin-to-edges-csv <edges.dat> <edges.csv>");
return;
}
let input = env::args().nth(2).unwrap();
let output = env::args().nth(3).unwrap();
match operation.unwrap().as_str() {
"--safes-json-to-edges-bin" => {
let safes = import_from_safes_json(&input);
let edges = safes.edges();
println!("Imported {} edges.", edges.edge_count());
write_edges_binary(edges, &output).unwrap();
println!("Export done.");
}
"--safes-json-to-edges-csv" => {
let safes = import_from_safes_json(&input);
let edges = safes.edges();
println!("Imported {} edges.", edges.edge_count());
write_edges_csv(edges, &output).unwrap();
println!("Export done.");
}
"--edges-csv-to-edges-bin" => {
let edges = &read_edges_csv(&input).unwrap();
println!("Imported {} edges.", edges.edge_count());
write_edges_binary(edges, &output).unwrap();
println!("Export done.");
}
"--edges-bin-to-edges-csv" => {
let edges = &read_edges_binary(&input).unwrap();
println!("Imported {} edges.", edges.edge_count());
write_edges_csv(edges, &output).unwrap();
println!("Export done.");
}
_ => unreachable!(),
}
}

View file

@ -627,7 +627,7 @@ mod test {
#[test] #[test]
fn trust_transfer_limit() { fn trust_transfer_limit() {
let (a, b, c, d, t1, t2) = addresses(); let (a, b, c, d, ..) = addresses();
let edges = build_edges(vec![ let edges = build_edges(vec![
// The following two edges should be balance-limited, // The following two edges should be balance-limited,
// i.e. a -> first intermediate is limited by the max of the two. // i.e. a -> first intermediate is limited by the max of the two.

View file

@ -1,6 +1,7 @@
use std::collections::BTreeSet;
use std::fs::File; use std::fs::File;
use std::io::Read;
use std::io::{self, BufRead}; use std::io::{self, BufRead};
use std::io::{Read, Write};
use std::{collections::HashMap, io::BufReader}; use std::{collections::HashMap, io::BufReader};
use crate::types::edge::EdgeDB; use crate::types::edge::EdgeDB;
@ -42,6 +43,28 @@ pub fn read_edges_csv(path: &String) -> Result<EdgeDB, io::Error> {
Ok(EdgeDB::new(edges)) Ok(EdgeDB::new(edges))
} }
pub fn write_edges_binary(edges: &EdgeDB, path: &String) -> Result<(), io::Error> {
let mut file = File::create(path)?;
let address_index = write_address_index(&mut file, edges)?;
write_edges(&mut file, edges, &address_index)
}
pub fn write_edges_csv(edges: &EdgeDB, path: &String) -> Result<(), io::Error> {
let mut file = File::create(path)?;
let mut sorted_edges = edges.edges().clone();
sorted_edges.sort();
for Edge {
from,
to,
token,
capacity,
} in sorted_edges
{
writeln!(file, "{from},{to},{token},{capacity}")?;
}
Ok(())
}
fn read_address_index(file: &mut File) -> Result<HashMap<u32, Address>, io::Error> { fn read_address_index(file: &mut File) -> Result<HashMap<u32, Address>, io::Error> {
let address_count = read_u32(file)?; let address_count = read_u32(file)?;
let mut addresses = HashMap::new(); let mut addresses = HashMap::new();
@ -53,18 +76,50 @@ fn read_address_index(file: &mut File) -> Result<HashMap<u32, Address>, io::Erro
Ok(addresses) Ok(addresses)
} }
fn write_address_index(
file: &mut File,
edges: &EdgeDB,
) -> Result<HashMap<Address, u32>, io::Error> {
let mut addresses = BTreeSet::new();
for Edge {
from, to, token, ..
} in edges.edges()
{
addresses.insert(*from);
addresses.insert(*to);
addresses.insert(*token);
}
write_u32(file, addresses.len() as u32)?;
let mut index = HashMap::new();
for (i, addr) in addresses.into_iter().enumerate() {
file.write_all(&addr.to_bytes())?;
index.insert(addr, i as u32);
}
Ok(index)
}
fn read_u32(file: &mut File) -> Result<u32, io::Error> { fn read_u32(file: &mut File) -> Result<u32, io::Error> {
let mut buf = [0; 4]; let mut buf = [0; 4];
file.read_exact(&mut buf)?; file.read_exact(&mut buf)?;
Ok(u32::from_be_bytes(buf)) Ok(u32::from_be_bytes(buf))
} }
fn write_u32(file: &mut File, v: u32) -> Result<(), io::Error> {
let buf = v.to_be_bytes();
file.write_all(&buf)
}
fn read_u8(file: &mut File) -> Result<u8, io::Error> { fn read_u8(file: &mut File) -> Result<u8, io::Error> {
let mut buf = [0; 1]; let mut buf = [0; 1];
file.read_exact(&mut buf)?; file.read_exact(&mut buf)?;
Ok(u8::from_be_bytes(buf)) Ok(u8::from_be_bytes(buf))
} }
fn write_u8(file: &mut File, v: u8) -> Result<(), io::Error> {
let buf = v.to_be_bytes();
file.write_all(&buf)
}
fn read_address( fn read_address(
file: &mut File, file: &mut File,
address_index: &HashMap<u32, Address>, address_index: &HashMap<u32, Address>,
@ -73,6 +128,14 @@ fn read_address(
Ok(address_index[&index]) Ok(address_index[&index])
} }
fn write_address(
file: &mut File,
address: &Address,
address_index: &HashMap<Address, u32>,
) -> Result<(), io::Error> {
write_u32(file, *address_index.get(address).unwrap())
}
fn read_u256(file: &mut File) -> Result<U256, io::Error> { fn read_u256(file: &mut File) -> Result<U256, io::Error> {
let length = read_u8(file)? as usize; let length = read_u8(file)? as usize;
let mut bytes = [0u8; 32]; let mut bytes = [0u8; 32];
@ -82,6 +145,16 @@ fn read_u256(file: &mut File) -> Result<U256, io::Error> {
Ok(U256::new(high, low)) Ok(U256::new(high, low))
} }
fn write_u256(file: &mut File, v: &U256) -> Result<(), io::Error> {
let v_bytes = v.to_bytes();
if v_bytes.is_empty() {
file.write_all(&[1, 0])
} else {
write_u8(file, v_bytes.len() as u8)?;
file.write_all(&v_bytes)
}
}
fn read_edges(file: &mut File, address_index: &HashMap<u32, Address>) -> Result<EdgeDB, io::Error> { fn read_edges(file: &mut File, address_index: &HashMap<u32, Address>) -> Result<EdgeDB, io::Error> {
let edge_count = read_u32(file)?; let edge_count = read_u32(file)?;
let mut edges = Vec::new(); let mut edges = Vec::new();
@ -100,6 +173,29 @@ fn read_edges(file: &mut File, address_index: &HashMap<u32, Address>) -> Result<
Ok(EdgeDB::new(edges)) Ok(EdgeDB::new(edges))
} }
fn write_edges(
file: &mut File,
edges: &EdgeDB,
address_index: &HashMap<Address, u32>,
) -> Result<(), io::Error> {
write_u32(file, edges.edge_count() as u32)?;
let mut sorted_edges = edges.edges().clone();
sorted_edges.sort();
for Edge {
from,
to,
token,
capacity,
} in &sorted_edges
{
write_address(file, from, address_index)?;
write_address(file, to, address_index)?;
write_address(file, token, address_index)?;
write_u256(file, capacity)?;
}
Ok(())
}
fn unescape(input: &str) -> &str { fn unescape(input: &str) -> &str {
match input.chars().next() { match input.chars().next() {
Some('"') | Some('\'') => { Some('"') | Some('\'') => {

View file

@ -1,4 +1,4 @@
use std::{collections::BTreeMap, default}; use std::collections::BTreeMap;
use crate::types::{edge::EdgeDB, Address, Edge, Safe, U256}; use crate::types::{edge::EdgeDB, Address, Edge, Safe, U256};
@ -21,6 +21,10 @@ impl DB {
db db
} }
pub fn edges(&self) -> &EdgeDB {
&self.edges
}
fn compute_edges(&mut self) { fn compute_edges(&mut self) {
let mut edges = vec![]; let mut edges = vec![];
for (user, safe) in &self.safes { for (user, safe) in &self.safes {
@ -44,7 +48,7 @@ impl DB {
// send tokens back to owner // send tokens back to owner
for (token, balance) in &safe.balances { for (token, balance) in &safe.balances {
if let Some(owner) = self.token_owner.get(token) { if let Some(owner) = self.token_owner.get(token) {
if *user != *owner { if *user != *owner && *balance != U256::from(0) {
edges.push(Edge { edges.push(Edge {
from: *user, from: *user,
to: *owner, to: *owner,

View file

@ -1,2 +1,2 @@
mod db; pub mod db;
mod safes_json; pub mod safes_json;

View file

@ -57,6 +57,7 @@ pub fn import_from_safes_json(file: &str) -> DB {
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
struct Safes<'a> { struct Safes<'a> {
#[allow(dead_code)]
block_number: &'a str, block_number: &'a str,
safes: Vec<JsonSafe<'a>>, safes: Vec<JsonSafe<'a>>,
} }
@ -76,6 +77,7 @@ struct JsonSafe<'a> {
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
struct Edge<'a> { struct Edge<'a> {
#[allow(dead_code)]
limit: Option<&'a str>, limit: Option<&'a str>,
limit_percentage: &'a str, limit_percentage: &'a str,
can_send_to_address: &'a str, can_send_to_address: &'a str,
@ -104,12 +106,3 @@ struct Token<'a> {
struct Owner<'a> { struct Owner<'a> {
id: &'a str, id: &'a str,
} }
mod test {
use super::*;
#[test]
pub fn t1() {
import_from_safes_json("/tmp/safes.json");
}
}

View file

@ -7,6 +7,10 @@ impl Address {
pub fn short(&self) -> String { pub fn short(&self) -> String {
format!("{self}")[..8].to_string() format!("{self}")[..8].to_string()
} }
pub fn to_bytes(self) -> [u8; 20] {
self.0
}
} }
impl Debug for Address { impl Debug for Address {

View file

@ -39,6 +39,10 @@ impl EdgeDB {
self.edges.len() self.edges.len()
} }
pub fn edges(&self) -> &Vec<Edge> {
&self.edges
}
pub fn update(&mut self, update: Edge) { pub fn update(&mut self, update: Edge) {
match self.index_of(&update) { match self.index_of(&update) {
Some(i) => self.edges[i].capacity = update.capacity, Some(i) => self.edges[i].capacity = update.capacity,

View file

@ -1,6 +1,8 @@
use super::Address; use super::Address;
pub struct Token { pub struct Token {
#[allow(dead_code)]
address: Address, address: Address,
#[allow(dead_code)]
owner: Address, owner: Address,
} }

View file

@ -29,6 +29,7 @@ impl U256 {
let value = BigUint::from(self.0[0]) << 128 | BigUint::from(self.0[1]); let value = BigUint::from(self.0[0]) << 128 | BigUint::from(self.0[1]);
format!("{value}") format!("{value}")
} }
pub fn to_decimal_fraction(self) -> String { pub fn to_decimal_fraction(self) -> String {
let value: BigUint = self.into(); let value: BigUint = self.into();
let formatted = format!("{value}"); let formatted = format!("{value}");
@ -46,6 +47,19 @@ impl U256 {
_ => "0+eps".to_string(), _ => "0+eps".to_string(),
} }
} }
pub fn to_bytes(&self) -> Vec<u8> {
let mut result = Vec::new();
for i in 0..=1 {
for j in (0..16).rev() {
let b = ((self.0[i] >> (j * 8)) & 0xff) as u8;
if b != 0 || !result.is_empty() {
result.push(b);
}
}
}
result
}
} }
impl From<u128> for U256 { impl From<u128> for U256 {
@ -279,4 +293,24 @@ mod test {
U256::from("0x8000000000000000000000000000000000000000000000000000000000000000") U256::from("0x8000000000000000000000000000000000000000000000000000000000000000")
); );
} }
#[test]
fn to_bytes() {
let zero = U256::from("0");
assert_eq!(zero.to_bytes(), Vec::<u8>::new());
assert_eq!(U256::from("2").to_bytes(), vec![2]);
assert_eq!(
U256::from("0x100000000000000000000000000000000").to_bytes(),
vec![1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
);
assert_eq!(
U256::from("0xff00000000000000000000000000000001").to_bytes(),
vec![255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]
);
assert_eq!(
U256::from("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
.to_bytes(),
vec![255; 32]
);
}
} }

View file

@ -1,14 +1,12 @@
use pathfinder2::graph::compute_flow; use pathfinder2::graph::compute_flow;
use pathfinder2::io::read_edges_binary; use pathfinder2::io::read_edges_binary;
use pathfinder2::types::edge::EdgeDB; use pathfinder2::types::edge::EdgeDB;
use pathfinder2::types::{Address, Edge, U256}; use pathfinder2::types::{Address, U256};
use std::collections::HashMap;
use std::process::Command; use std::process::Command;
const HUB_ADDRESS: &'static str = "0x29b9a7fBb8995b2423a71cC17cf9810798F6C543"; const HUB_ADDRESS: &str = "0x29b9a7fBb8995b2423a71cC17cf9810798F6C543";
const TRANSFER_THROUGH_SIG: &'static str = const TRANSFER_THROUGH_SIG: &str = "transferThrough(address[],address[],address[],uint256[])";
"transferThrough(address[],address[],address[],uint256[])"; const RPC_URL: &str = "https://rpc.gnosischain.com";
const RPC_URL: &'static str = "https://rpc.gnosischain.com";
#[test] #[test]
fn test_flow_chris_martin() { fn test_flow_chris_martin() {
@ -64,7 +62,7 @@ fn test_flow(
max_distance: Option<u64>, max_distance: Option<u64>,
) { ) {
let transfers = compute_flow(source, sink, edges, requested_flow, max_distance); let transfers = compute_flow(source, sink, edges, requested_flow, max_distance);
println!("{:?}", transfers); println!("{transfers:?}");
let token_owners = transfers let token_owners = transfers
.1 .1