Add serialization for Analyzed<T> and pipeline resume

Fix pipeline name from file name with suffix

Variable naming

Replace `splitted` with `split`

Co-authored-by: Leo <[email protected]>

Fix clippy

Add test for serde of PIL

Changed the optimized PIL file extension to .pilo

Changed function names to reflect the operation on pil object

ast/Cargo.toml

@@ -16,6 +16,7 @@ num-traits = "0.2.15"
 diff = "0.1"
 log = "0.4.18"
 derive_more = "0.99.17"
+serde = { version = "1.0", default-features = false, features = ["alloc", "derive", "rc"] }
 
 [dev-dependencies]
 pretty_assertions = "1.3.0"

ast/src/analyzed/mod.rs

@@ -8,6 +8,7 @@ use std::fmt::Display;
 use std::ops::{self, ControlFlow};
 
 use powdr_number::{DegreeType, FieldElement};
+use serde::{Deserialize, Serialize};
 
 use crate::parsed::utils::expr_any;
 use crate::parsed::visitor::ExpressionVisitable;
@@ -18,7 +19,7 @@ use crate::SourceRef;
 
 use self::types::TypedExpression;
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum StatementIdentifier {
     /// Either an intermediate column or a definition.
     Definition(String),
@@ -27,7 +28,7 @@ pub enum StatementIdentifier {
     Identity(usize),
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Analyzed<T> {
     /// The degree of all namespaces, which must match. If there are no namespaces, then `None`.
     pub degree: Option<DegreeType>,
@@ -404,7 +405,7 @@ fn inlined_expression_from_intermediate_poly_id<T: Copy + Display>(
     expr
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Symbol {
     pub id: u64,
     pub source: SourceRef,
@@ -461,7 +462,7 @@ impl Symbol {
 
 /// The "kind" of a symbol. In the future, this will be mostly
 /// replaced by its type.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
 pub enum SymbolKind {
     /// Fixed, witness or intermediate polynomial
     Poly(PolynomialType),
@@ -472,15 +473,15 @@ pub enum SymbolKind {
     Other(),
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum FunctionValueDefinition<T> {
     Array(Vec<RepeatedArray<T>>),
     Query(Expression<T>),
     Expression(TypedExpression<T>),
 }
 
 /// An array of elements that might be repeated.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct RepeatedArray<T> {
     /// The pattern to be repeated
     pattern: Vec<Expression<T>>,
@@ -520,7 +521,7 @@ impl<T> RepeatedArray<T> {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct PublicDeclaration {
     pub id: u64,
     pub source: SourceRef,
@@ -540,7 +541,7 @@ impl PublicDeclaration {
     }
 }
 
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
 pub struct Identity<Expr> {
     /// The ID is specific to the identity kind.
     pub id: u64,
@@ -579,7 +580,7 @@ impl<T> Identity<AlgebraicExpression<T>> {
     }
 }
 
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Serialize, Deserialize)]
 pub enum IdentityKind {
     Polynomial,
     Plookup,
@@ -600,13 +601,13 @@ impl<T> SelectedExpressions<AlgebraicExpression<T>> {
 
 pub type Expression<T> = parsed::Expression<T, Reference>;
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum Reference {
     LocalVar(u64, String),
     Poly(PolynomialReference),
 }
 
-#[derive(Debug, Clone, Eq)]
+#[derive(Debug, Clone, Eq, Serialize, Deserialize)]
 pub struct AlgebraicReference {
     /// Name of the polynomial - just for informational purposes.
     /// Comparisons are based on polynomial ID.
@@ -653,7 +654,7 @@ impl Hash for AlgebraicReference {
         self.next.hash(state);
     }
 }
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub enum AlgebraicExpression<T> {
     Reference(AlgebraicReference),
     PublicReference(String),
@@ -667,7 +668,7 @@ pub enum AlgebraicExpression<T> {
     UnaryOperation(AlgebraicUnaryOperator, Box<AlgebraicExpression<T>>),
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize, Deserialize)]
 pub enum AlgebraicBinaryOperator {
     Add,
     Sub,
@@ -703,7 +704,7 @@ impl TryFrom<BinaryOperator> for AlgebraicBinaryOperator {
     }
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize, Deserialize)]
 pub enum AlgebraicUnaryOperator {
     Minus,
 }
@@ -790,7 +791,7 @@ impl<T> From<T> for AlgebraicExpression<T> {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct PolynomialReference {
     /// Name of the polynomial - just for informational purposes.
     /// Comparisons are based on polynomial ID.
@@ -801,7 +802,7 @@ pub struct PolynomialReference {
     pub poly_id: Option<PolyID>,
 }
 
-#[derive(Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)]
+#[derive(Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct PolyID {
     pub id: u64,
     pub ptype: PolynomialType,
@@ -819,7 +820,7 @@ impl From<&Symbol> for PolyID {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
 pub enum PolynomialType {
     Committed,
     Constant,

ast/src/analyzed/types.rs

@@ -1,18 +1,19 @@
 use std::fmt::Display;
 
 use powdr_number::FieldElement;
+use serde::{Deserialize, Serialize};
 
 use crate::parsed::{ArrayTypeName, Expression, FunctionTypeName, TupleTypeName, TypeName};
 
 use super::Reference;
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub struct TypedExpression<T, Ref = Reference> {
     pub e: Expression<T, Ref>,
     pub ty: Option<Type>,
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub enum Type {
     /// Boolean
     Bool,
@@ -71,7 +72,7 @@ impl<T: FieldElement, Ref: Display> From<TypeName<Expression<T, Ref>>> for Type
     }
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub struct ArrayType {
     pub base: Box<Type>,
     pub length: Option<u64>,
@@ -95,7 +96,7 @@ impl<T: FieldElement, Ref: Display> From<ArrayTypeName<Expression<T, Ref>>> for
     }
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub struct TupleType {
     pub items: Vec<Type>,
 }
@@ -108,7 +109,7 @@ impl<T: FieldElement, Ref: Display> From<TupleTypeName<Expression<T, Ref>>> for
     }
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub struct FunctionType {
     pub params: Vec<Type>,
     pub value: Box<Type>,

ast/src/lib.rs

@@ -2,6 +2,7 @@
 
 use itertools::Itertools;
 use log::log_enabled;
+use serde::{Deserialize, Serialize};
 use std::fmt::{Display, Result, Write};
 use std::sync::Arc;
 
@@ -21,7 +22,7 @@ pub struct DiffMonitor {
     current: Option<String>,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
 pub struct SourceRef {
     pub file: Option<Arc<str>>,
     pub line: usize,

ast/src/parsed/mod.rs

@@ -11,6 +11,7 @@ use std::{
 };
 
 use powdr_number::{DegreeType, FieldElement};
+use serde::{Deserialize, Serialize};
 
 use self::asm::{Part, SymbolPath};
 use crate::SourceRef;
@@ -151,7 +152,7 @@ impl<T> PilStatement<T> {
     }
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub struct SelectedExpressions<Expr> {
     pub selector: Option<Expr>,
     pub expressions: Vec<Expr>,
@@ -178,7 +179,7 @@ impl<Expr> SelectedExpressions<Expr> {
     }
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub enum Expression<T, Ref = NamespacedPolynomialReference> {
     Reference(Ref),
     PublicReference(String),
@@ -292,18 +293,18 @@ impl NamespacedPolynomialReference {
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
 pub struct LambdaExpression<T, Ref = NamespacedPolynomialReference> {
     pub params: Vec<String>,
     pub body: Box<Expression<T, Ref>>,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
 pub struct ArrayLiteral<T, Ref = NamespacedPolynomialReference> {
     pub items: Vec<Expression<T, Ref>>,
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize, Deserialize)]
 pub enum UnaryOperator {
     Minus,
     LogicalNot,
@@ -320,7 +321,7 @@ impl UnaryOperator {
     }
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize, Deserialize)]
 pub enum BinaryOperator {
     Add,
     Sub,
@@ -344,32 +345,32 @@ pub enum BinaryOperator {
     Greater,
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub struct IndexAccess<T, Ref = NamespacedPolynomialReference> {
     pub array: Box<Expression<T, Ref>>,
     pub index: Box<Expression<T, Ref>>,
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub struct FunctionCall<T, Ref = NamespacedPolynomialReference> {
     pub function: Box<Expression<T, Ref>>,
     pub arguments: Vec<Expression<T, Ref>>,
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub struct MatchArm<T, Ref = NamespacedPolynomialReference> {
     pub pattern: MatchPattern<T, Ref>,
     pub value: Expression<T, Ref>,
 }
 
 /// A pattern for a match arm. We could extend this in the future.
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub enum MatchPattern<T, Ref = NamespacedPolynomialReference> {
     CatchAll,
     Pattern(Expression<T, Ref>),
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
 pub struct IfExpression<T, Ref = NamespacedPolynomialReference> {
     pub condition: Box<Expression<T, Ref>>,
     pub body: Box<Expression<T, Ref>>,

cli/src/main.rs

@@ -812,7 +812,7 @@ fn read_and_prove<T: FieldElement>(
     params: Option<String>,
 ) -> Result<(), Vec<String>> {
     Pipeline::<T>::default()
-        .from_file(file.to_path_buf())
+        .from_maybe_pil_object(file.to_path_buf())
         .with_output(dir.to_path_buf(), true)
         .read_generated_witness(dir)
         .with_setup_file(params.map(PathBuf::from))

number/Cargo.toml

@@ -12,9 +12,12 @@ ark-bn254 = { version = "0.4.0", default-features = false, features = [
     "scalar_field",
 ] }
 ark-ff = "0.4.2"
+ark-serialize = "0.4.2"
 num-bigint = "0.4.3"
 num-traits = "0.2.15"
 csv = "1.3"
+serde = { version = "1.0", default-features = false, features = ["alloc", "derive", "rc"] }
+serde_with = "3.6.1"
 
 [dev-dependencies]
 test-log = "0.2.12"

number/src/bn254.rs

@@ -1,4 +1,6 @@
 use ark_bn254::Fr;
+use serde::{Deserialize, Serialize};
+
 powdr_field!(Bn254Field, Fr);
 
 #[cfg(test)]

number/src/goldilocks.rs

@@ -1,4 +1,5 @@
 use ark_ff::{Fp64, MontBackend, MontConfig};
+use serde::{Deserialize, Serialize};
 
 #[derive(MontConfig)]
 #[modulus = "18446744069414584321"]

number/src/macros.rs

@@ -11,8 +11,24 @@ macro_rules! powdr_field {
         use std::ops::*;
         use std::str::FromStr;
 
-        #[derive(Clone, Copy, PartialEq, Eq, Debug, Default, PartialOrd, Ord, Hash)]
+        #[derive(
+            Clone,
+            Copy,
+            PartialEq,
+            Eq,
+            Debug,
+            Default,
+            PartialOrd,
+            Ord,
+            Hash,
+            Serialize,
+            Deserialize,
+        )]
         pub struct $name {
+            #[serde(
+                serialize_with = "crate::serialize::ark_se",
+                deserialize_with = "crate::serialize::ark_de"
+            )]
             value: $ark_type,
         }