add evaluation.py to ml branch

.github/workflows/ci.yml

@@ -12,43 +12,66 @@ jobs:
   notify-about-push:
     runs-on: ubuntu-latest
     steps:
-    - name: Send msg on push
-      uses: appleboy/telegram-action@master
-      with:
-        to: ${{ secrets.TELEGRAM_TO }}
-        token: ${{ secrets.TELEGRAM_TOKEN }}
-        format: html
-        args: |
+      - name: Send msg on push
+        uses: appleboy/telegram-action@master
+        with:
+          to: ${{ secrets.TELEGRAM_TO }}
+          token: ${{ secrets.TELEGRAM_TOKEN }}
+          format: html
+          args: |
             Repository: <strong>${{ github.event.repository.full_name }}</strong>
             Ref: <strong>${{ github.ref }}</strong>
             Event: <strong>${{ github.event_name }}</strong>
             Info:  ${{ github.event.pull_request.title }}
             ${{ github.event.pull_request.html_url }}
 
-  nitta:
+  nitta-haskell-deps:
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 45
     steps:
       - uses: actions/checkout@v3
 
       - name: Cache haskell-stack
         uses: actions/[email protected]
         with:
-          path: ~/.stack
+          path: |
+            ~/.stack
+            .stack-work
           key: ${{ runner.os }}-haskell-stack-${{ hashFiles('**/stack.yaml', '**/package.yaml') }}
 
       - name: Install haskell-stack
         uses: haskell/actions/[email protected]
         with:
           enable-stack: true
           stack-no-global: true
-          stack-version: 'latest'
+          stack-version: "latest"
 
       - name: Build nitta backend dependencies and doctest
         run: |
           stack build --haddock --test --only-dependencies
           stack install doctest
 
+  nitta:
+    runs-on: ubuntu-latest
+    needs: nitta-haskell-deps
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Cache haskell-stack
+        uses: actions/[email protected]
+        with:
+          path: |
+            ~/.stack
+            .stack-work
+          key: ${{ runner.os }}-haskell-stack-${{ hashFiles('**/stack.yaml', '**/package.yaml') }}
+
+      - name: Install haskell-stack
+        uses: haskell/actions/[email protected]
+        with:
+          enable-stack: true
+          stack-no-global: true
+          stack-version: "latest"
       - name: Install Icarus Verilog
         run: sudo apt-get install iverilog
 
@@ -59,17 +82,15 @@ jobs:
         run: stack hpc report nitta
 
       - name: Check examples by doctest
-        run: |
-          stack build
-          find src -name '*.hs' -exec grep -l '>>>' {} \; | xargs -t -L 1 -P 4 stack exec doctest
+        run: find src -name '*.hs' -exec grep -l '>>>' {} \; | xargs -t -L 1 -P 4 stack exec doctest
 
       - name: Generate backend API
         run: stack exec nitta-api-gen -- -v
 
       - name: Cache node_modules
         uses: actions/[email protected]
         with:
-          path: '**/node_modules'
+          path: "**/node_modules"
           key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
 
       - name: Build nitta frontend dependencies
@@ -88,7 +109,7 @@ jobs:
 
       - name: Copy test coverage to GH_PAGES_DIR
         run: cp -r $(stack path --local-hpc-root)/combined/custom ${{ env.GH_PAGES_DIR }}/hpc
-      
+
       - name: Copy API doc to GH_PAGES_DIR
         run: |
           mkdir -p "${{ env.GH_PAGES_DIR }}/rest-api/"
@@ -106,7 +127,6 @@ jobs:
 
   verilog-formatting:
     runs-on: ubuntu-latest
-    needs: nitta
     container: ryukzak/alpine-iverilog
     defaults:
       run:
@@ -120,43 +140,58 @@ jobs:
 
   haskell-lint:
     runs-on: ubuntu-latest
-    needs: nitta
     steps:
-    - uses: actions/checkout@v3
+      - uses: actions/checkout@v3
 
-    - name: 'Set up HLint'
-      uses: haskell/actions/[email protected]
-      with:
-        version: '3.5'
+      - name: "Set up HLint"
+        uses: haskell/actions/[email protected]
+        with:
+          version: "3.5"
 
-    - name: 'Run HLint'
-      uses: haskell/actions/[email protected]
-      with:
-        path: .
-        fail-on: suggestion
+      - name: "Run HLint"
+        uses: haskell/actions/[email protected]
+        with:
+          path: .
+          fail-on: suggestion
 
   haskell-formatting:
     runs-on: ubuntu-latest
-    needs: nitta
     steps:
       - uses: actions/checkout@v3
 
-      - name: Cache haskell-stack and fourmolu
-        uses: actions/[email protected]
-        with:
-          path: ~/.stack
-          key: ${{ runner.os }}-haskell-stack-${{ hashFiles('**/stack.yaml', '**/package.yaml') }}-fourmolu
-
       - name: Check formatting
-        uses: fourmolu/fourmolu-action@v6
+        uses: fourmolu/fourmolu-action@v8 # fourmolu-0.12.0.0
 
   typescript-formatting:
     runs-on: ubuntu-latest
-    needs: nitta
     steps:
       - uses: actions/checkout@v3
+
       - name: Check ts and tsx code style by prettier
         working-directory: ./web
         run: |
           yarn add -s prettier
           yarn exec -s prettier -- --check src/**/*.{ts,tsx}
+
+  python-formatting:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+
+      - name: Install dependencies
+        run: |
+          pip install black isort flake8
+
+      - name: Check code style with black
+        run: black --check ml
+
+      - name: Sort imports with isort
+        run: isort --recursive ml
+
+      - name: Check code style with flake8
+        run: flake8 ml

README.md

@@ -44,12 +44,12 @@ See: [CONTRIBUTING.md](CONTRIBUTING.md)
 Install [Stack](https://github.com/commercialhaskell/stack) and required developer tools for Haskell.
 
 ``` console
-$ brew install stack
-$ stack install hlint fourmolu
+$ brew install ghcup
+$ ghcup install stack
 ```
 > Make sure that PATH contains $HOME/.local/bin.
 
-> Make sure that you have up to date version of hlint and fourmolu (as on CI)!   
+> Make sure that you have up to date version of hlint and fourmolu (as on CI)!
 
 Install [icarus-verilog](https://github.com/steveicarus/iverilog/) and [gtkwave](https://github.com/gtkwave/gtkwave).
 ``` console
@@ -73,7 +73,7 @@ $ stack install hlint fourmolu
 ```
 > Make sure that PATH contains $HOME/.local/bin.
 
-> Make sure that you have up to date version of hlint and fourmolu (as on CI)!  
+> Make sure that you have up to date version of hlint and fourmolu (as on CI)!
 
 Install [icarus-verilog](https://github.com/steveicarus/iverilog/) and [gtkwave](https://github.com/gtkwave/gtkwave).
 ``` console
@@ -257,4 +257,3 @@ $ stack exec nitta -- examples/teacup.lua -v --lsim -t=fx24.32
 $ stack exec nitta -- examples/teacup.lua -p=8080
 Running NITTA server at http://localhost:8080 ...
 ```
-

app/APIGen.hs

@@ -76,8 +76,8 @@ $(deriveTypeScript defaultOptions ''OptimizeAccumMetrics)
 $(deriveTypeScript defaultOptions ''ResolveDeadlockMetrics)
 
 $(deriveTypeScript defaultOptions ''ViewPointID)
-$(deriveTypeScript defaultOptions ''TimelinePoint)
 $(deriveTypeScript defaultOptions ''Interval)
+$(deriveTypeScript defaultOptions ''TimelinePoint)
 $(deriveTypeScript defaultOptions ''TimeConstraint)
 $(deriveTypeScript defaultOptions ''TimelineWithViewPoint)
 $(deriveTypeScript defaultOptions ''ProcessTimelines)
@@ -109,8 +109,8 @@ $(deriveTypeScript defaultOptions ''TestbenchReport)
 
 -- Microarchitecture
 $(deriveTypeScript defaultOptions ''IOSynchronization)
-$(deriveTypeScript defaultOptions ''NetworkDesc)
 $(deriveTypeScript defaultOptions ''UnitDesc)
+$(deriveTypeScript defaultOptions ''NetworkDesc)
 $(deriveTypeScript defaultOptions ''MicroarchitectureDesc)
 
 main = do
@@ -186,8 +186,9 @@ main = do
             (ts ++ "\n" ++ "type NId = string\n")
             [ ("type ", "export type ") -- export all types
             , ("interface ", "export interface ") -- export all interfaces
-            , ("[k: T1]", "[k: string]") -- dirty hack for fixing map types for TestbenchReport
-            , ("[k: T2]", "[k: string]") -- dirty hack for fixing map types for TestbenchReport
+            , ("[k in T1]?", "[k: string]") -- dirty hack for fixing map types for TestbenchReport
+            , ("[k in T2]?", "[k: string]") -- dirty hack for fixing map types for TestbenchReport
+            , ("[k in number]?: number", "[k: number]: number") -- dirty hack for fixing map types for TreeInfo
             ]
     infoM "NITTA.APIGen" $ "Generate typescript interface " <> output_path <> "/types.ts...OK"
 

app/Main.hs

@@ -23,6 +23,7 @@ import Data.ByteString.Lazy.Char8 qualified as BS
 import Data.Default (def)
 import Data.Maybe
 import Data.Proxy
+
 import Data.String.Utils qualified as S
 import Data.Text qualified as T
 import Data.Text.IO qualified as T
@@ -171,6 +172,8 @@ getNittaArgs = do
             exitWith exitCode
     catch (cmdArgs nittaArgs) handleError
 
+fromConf toml s = getFromTomlSection s =<< toml
+
 main = do
     ( Nitta
             filename
@@ -197,7 +200,6 @@ main = do
         Nothing -> return Nothing
         Just path -> Just . getToml <$> T.readFile path
 
-    let fromConf s = getFromTomlSection s =<< toml
     let exactFrontendType = identifyFrontendType filename frontend_language
 
     src <- readSourceCode filename
@@ -207,14 +209,15 @@ main = do
                 -- FIXME: https://nitta.io/nitta-corp/nitta/-/issues/50
                 -- data for sin_ident
                 received = [("u#0", map (\i -> read $ show $ sin ((2 :: Double) * 3.14 * 50 * 0.001 * i)) [0 .. toEnum n])]
-                ioSync = fromJust $ io_sync <|> fromConf "ioSync" <|> Just Sync
+                ioSync = fromJust $ io_sync <|> fromConf toml "ioSync" <|> Just Sync
                 confMa = toml >>= Just . mkMicroarchitecture ioSync
+                ma :: BusNetwork T.Text T.Text (Attr (FX m b)) Int
                 ma
                     | auto_uarch && isJust confMa =
                         error $
                             "auto_uarch flag means that an empty uarch with default prototypes will be used. "
                                 <> "Remove uarch flag or specify prototypes list in config file and remove auto_uarch."
-                    | auto_uarch = microarchWithProtos ioSync :: BusNetwork T.Text T.Text (Attr (FX m b)) Int
+                    | auto_uarch = microarchWithProtos ioSync
                     | isJust confMa = fromJust confMa
                     | otherwise = defMicroarch ioSync
 
@@ -235,28 +238,30 @@ main = do
 
             prj <-
                 synthesizeTargetSystem
-                    def
+                    (def :: TargetSynthesis T.Text T.Text (Attr (FX m b)) Int)
                         { tName = "main"
                         , tPath = output_path
                         , tMicroArch = ma
                         , tDFG = frDataFlow
                         , tReceivedValues = received
                         , tTemplates = S.split ":" templates
+                        , tSynthesisMethod = stateOfTheArtSynthesisIO ()
                         , tSimulationCycleN = n
                         , tSourceCodeType = exactFrontendType
                         }
-                    >>= \case
-                        Left msg -> error msg
-                        Right p -> return p
+                    >>= either error return
 
             when lsim $ logicalSimulation format frPrettyLog prj
         )
         $ parseFX . fromJust
-        $ type_ <|> fromConf "type" <|> Just "fx32.32"
+        $ type_ <|> fromConf toml "type" <|> Just "fx32.32"
 
 parseFX input =
     let typePattern = mkRegex "fx([0-9]+).([0-9]+)"
-        [m, b] = fromMaybe (error "incorrect Bus type input") $ matchRegex typePattern input
+        (m, b) = case fromMaybe (error "incorrect Bus type input") $ matchRegex typePattern input of
+            [m_, b_] -> (m_, b_)
+            _ -> error "parseFX: impossible"
+
         convert = fromJust . someNatVal . read
      in (convert m, convert b)
 
@@ -281,6 +286,7 @@ readSourceCode filename = do
     return src
 
 -- | Simulation on intermediate level (data-flow graph)
+functionalSimulation :: (Val x, Var v) => Int -> [(v, [x])] -> [Char] -> FrontendResult v x -> IO ()
 functionalSimulation n received format FrontendResult{frDataFlow, frPrettyLog} = do
     let cntx = simulateDataFlowGraph n def received frDataFlow
     infoM "NITTA" "run functional simulation..."

fourmolu.yaml

@@ -6,3 +6,6 @@ diff-friendly-import-export: true # 'false' uses Ormolu-style lists
 respectful: true # don't be too opinionated about newlines etc.
 haddock-style: multi-line # '--' vs. '{-'
 newlines-between-decls: 1 # number of newlines between top-level declarations
+
+# this will be available only since fourmolu-0.12.0.0, see https://github.com/ryukzak/nitta/issues/242
+single-constraint-parens: never # always | never | auto (preserve as is) - whether to style optional parentheses around single constraints

ml/synthesis/README.md

@@ -147,6 +147,29 @@ docker run \
 	-it \
 	nitta-dev
 ```
+### Script evaluation guide
+
+The `evaluation.py` script supports various command-line arguments that can be used to customize and control the evaluation process. This guide provides an overview of the available arguments and their usage.
+
+- `example_paths` (required): Paths to the example files, separated by spaces.
+- `--evaluator` (optional): Evaluation methods to use. You can specify one or multiple methods, separated by spaces. Allowed values: nitta, ml.
+- `--nitta_args` (optional): Arguments passed to Nitta. Enter the arguments in the format `--nitta_args="<arguments>"`.
+- `--help`: Prints help information about the available arguments.
+
+examples using
+```bash
+python3 ml/synthesis/src/scripts/evaluation.py examples/fibonacci.lua examples/counter.lua --evaluator nitta ml --nitta_args="--format=csv"
+
+Algorithm: examples/fibonacci.lua
+     duration  depth  evaluator_calls      time
+nitta       5      8                9  0.906887
+ml          5      9               10  1.902110
+
+Algorithm: examples/counter.lua
+     duration  depth  evaluator_calls      time
+nitta       5      8                9  5.687186
+ml          5      8                9  7.122119
+```
 
 #### Linux
 

ml/synthesis/pyproject.toml

@@ -0,0 +1,8 @@
+[tool.flake8]
+max-line-length = 125
+
+[tool.isort]
+profile = "black"
+
+[tool.black]
+line-length = 12