convert setup_container and setup_postgres to bash scripts, and

CI/Docker builds work!

Dockerfile

@@ -1,16 +1,20 @@
-ARG BASE_IMAGE="ubuntu:20.04"
+ARG BASE_IMAGE="ubuntu:22.04"
 
 ### BASE IMAGE
 FROM $BASE_IMAGE as base
 ARG BASE_IMAGE
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV LLVM_DIR=/usr/lib/llvm-11
+ENV PATH="/scripts:${PATH}"
+ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
 
 # Copy dependencies lists into container. We copy them all and then do a mv because
 # we need to transform base_image into a windows compatible filename which we can't
 # do in a COPY command.
 COPY ./dependencies/* /tmp
+COPY ./requirements.txt /tmp
+
 RUN mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_build.txt /tmp/build_dep.txt && \
     mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_base.txt /tmp/base_dep.txt
 
@@ -21,9 +25,9 @@ RUN [ -e /tmp/base_dep.txt ] && \
     apt-get clean
 
 # Finally: Install panda debian package, you need a version that has the Dwarf2 Plugin
-RUN wget https://github.com/panda-re/panda/releases/download/v1.8.23/pandare_20.04.deb
-RUN command apt install -qq -y ./pandare_20.04.deb
-RUN pip install pandare
+RUN curl -LJO https://github.com/panda-re/panda/releases/download/v1.8.23/pandare_22.04.deb
+RUN apt install -qq -y ./pandare_22.04.deb
+RUN pip install -r /tmp/requirements.txt
 
 ### BUILD IMAGE - STAGE 2
 FROM base AS builder
@@ -38,13 +42,16 @@ RUN [ -e /tmp/build_dep.txt ] && \
 FROM builder as developer
 
 COPY ./tools/ /tools
-COPY setup_container.py /
+COPY ./scripts /scripts
+
+# Essentially same as setup_container.sh
+RUN cd /tools/btrace && ./compile.sh
 
-RUN python3 setup_container.py
+RUN rm -rf /tools/build
+RUN mkdir -p /tools/build
+RUN mkdir -p /tools/install
 
-# RUN cd /tools && \
-#    cmake -Bbuild -H. -DLLVM_DIR=$LLVM_DIR/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=/tools/install
+RUN cmake -B"/tools/build" -H"/tools" -DCMAKE_INSTALL_PREFIX="/tools/install"
+RUN make --no-print-directory -j4 install -C "/tools/build/lavaTool"
 
-# RUN cd /tools/build && \
-#    make && \
-#    make install V=1
+RUN make --no-print-directory -j4 install -C "/tools/build/fbi"

docs/setup.md

@@ -0,0 +1,24 @@
+# Setup LAVA
+
+## Ubuntu 64-bit 16.04.4
+The following install steps worked on 6/29/2019 with LAVA commit [c55bf1826ef9855a621f2652b30f16ac75b19cb6](https://github.com/panda-re/lava/commit/c55bf1826ef9855a621f2652b30f16ac75b19cb6).
+
+- Download and install [Ubuntu 64-bit 16.04.6](http://releases.ubuntu.com/16.04/ubuntu-16.04.6-desktop-amd64.iso) (SHA1 A09607901183AB25C675626024AA402663FA2558, MD5: 5416371CC0E990871746DDAAC89F1A5E).
+- `sudo add-apt-repository ppa:phulin/panda`
+- `sudo cp /etc/apt/sources.list /etc/apt/sources.list~`
+- `sudo sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list`
+- `sudo apt-get update`
+- `sudo apt-get install python-pip git protobuf-compiler protobuf-c-compiler libprotobuf-c0-dev libprotoc-dev python-protobuf libelf-dev libcapstone-dev libdwarf-dev python-pycparser llvm-3.3 clang-3.3 libc++-dev libwiretap-dev libwireshark-dev odb`
+- `sudo apt-get build-dep qemu`
+- `pip install colorama`
+- `cd ~/Desktop`
+- `git clone https://github.com/panda-re/lava.git`
+- `cd ~/Desktop/lava`
+- `python2 setup.py`
+
+## Updated
+
+- build Docker container with `docker/Dockerfile`
+- use `scripts/docker-shell.sh` to enter docker container
+- inside Docker container, run `python setup_container.py` to build lavaTools
+- exit Docker container, and run `python setup_postgres.py` and `init-host.py`

init-host.py

@@ -18,13 +18,11 @@
 from colorama import Fore
 from colorama import Style
 
-QCOW_URL = "http://panda.moyix.net/~moyix/wheezy_panda2.qcow2"
 # if moyix server is down, this image will also work
-# QCOW_URL = "https://panda.re/qcows/linux/debian/7.3/x86/debian_7.3_x86.qcow"
+QCOW_URL = "https://panda.re/qcows/linux/debian/7.3/x86/debian_7.3_x86.qcow"
 QCOW_FILE_NAME = "wheezy_panda2.qcow2"
 TAR_URL = "ftp://ftp.astron.com/pub/file/file-5.22.tar.gz"
 LAVA_DIR = dirname(abspath(sys.argv[0]))
-os.chdir(LAVA_DIR)
 
 
 def progress(msg):
@@ -57,20 +55,7 @@ def run(cmd):
 
 
 def main():
-    # try to import lava.mak as a config file if not exit
-    try:
-        def_lines = (line.strip() for line in open("lava.mak", "r")
-                     if not line.strip().startswith("#")
-                     and line.strip() != "")
-        def_lines = (line.split(":=") for line in def_lines)
-        def_lines = ((line[0].strip(), line[1].strip()) for line in def_lines)
-        LAVA_CONFS = dict(def_lines)
-        PANDA_BUILD_DIR = LAVA_CONFS["PANDA_BUILD_DIR"]
-        PANDA_BUILD_DIR = expandvars(PANDA_BUILD_DIR)
-        print("PANDA_BUILD_DIR Used {}".format(PANDA_BUILD_DIR))
-    except Exception:
-        error("Make sure to have properly configured lava.mak \
-              generated by setup.py")
+
     # parser = argparse.ArgumentParser(description='Setup LAVA')
     # parser.add_argument('-s', '--skip_docker_build', action='store_true',
     # default = False,
@@ -93,7 +78,7 @@ def main():
     else:
         progress("Found existing target_bins/{}".format(basename(TAR_URL)))
 
-    if not isfile(join(LAVA_DIR, basename(QCOW_URL))):
+    if not isfile(join(LAVA_DIR, basename(QCOW_FILE_NAME))):
         progress("Downloading {}".format(basename(QCOW_URL)))
         run(["wget", "--no-check-certificate", QCOW_URL, "-O", QCOW_FILE_NAME])
     else:
@@ -103,8 +88,7 @@ def main():
         progress("Building host.json")
         # Build host.json
         json_configs = {}
-        json_configs["qemu"] = join(join(PANDA_BUILD_DIR, "i386-softmmu"),
-                                    "qemu-system-i386")
+        json_configs["qemu"] = "panda-system-i386"
         json_configs["qcow_dir"] = LAVA_DIR
         json_configs["output_dir"] = join(LAVA_DIR, "target_injections")
         json_configs["config_dir"] = join(LAVA_DIR, "target_configs")
@@ -119,15 +103,7 @@ def main():
     else:
         progress("Found existing host.json")
 
-    # progress("(re)building the fbi")
-    # os.chdir(join(LAVA_DIR, "tools", "build", "fbi"))
-    # run(["make", "install", "-j4"])
-
-    # progress("(re)building lavaTool")
-    # os.chdir(join(LAVA_DIR, "tools", "build", "lavaTool"))
-    # run(["./compile-on-docker.sh"])
-
-    progress("Sucessful!  Now run:\n  $ scripts/lava.sh -ak file")
+    progress("Successful!  Now run:\n  $ scripts/lava.sh -ak file")
     return 0
 
 

install.sh

@@ -13,15 +13,6 @@ progress() {
   echo -e "\e[32m[lava_install]\e[0m \e[1m$1\e[0m"
 }
 
-# Step 1: Install panda debian package
-wget https://github.com/panda-re/panda/releases/download/v1.8.23/pandare_22.04.deb
-$SUDO apt install ./pandare_22.04.deb
-
-# Remove the comment to update all debian stuff from sources.list
-$SUDO cp /etc/apt/sources.list /etc/apt/sources.list~
-$SUDO sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list
-$SUDO apt-get update
-
 progress "Updates complete"
 
 # This was originally in the docs/setup.md, I removed things starting with 'python-' as that should be installed via pip
@@ -53,12 +44,10 @@ $SUDO apt install ./pandare_22.04.deb
 
 progress "Installed build dependencies"
 
-$SUDO pip3 install --upgrade pip
-
-# This seems to be the better replacement to have all python packages
-$SUDO pip3 install -r docker/requirements.txt
+pip3 install --upgrade pip
+pip3 install -r requirements.txt
 progress "Installed Python requirements"
 
-$SUDO python3 setup_container.py
+$SUDO bash ./setup_container.sh
 
 progress "Installed LAVA"

requirements.txt

@@ -0,0 +1,9 @@
+colorama
+db
+ipython
+lockfile
+numpy
+pandare
+PyYAML
+SQLAlchemy
+tabulate

scripts/manual_lava.py

@@ -1,12 +1,6 @@
-import re
 import sys
-import random
 import subprocess as sp
 import sys
-
-# ok this is gross
-sys.path.append("/home/tleek/git/panda-leet/panda/scripts")
-
 from pandare.plog_reader import PLogReader
 
 #

scripts/setup_postgres.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -ex
+
+PGPASS="${HOME}/.pgpass"
+
+if [ ! -f "${PGPASS}" ]; then
+    postgres_depends=$(dpkg-query -W -f='${depends}' 'postgresql')
+    postgres_pkg=$(echo "${postgres_depends}" | grep -oP 'postgresql-[0-9]+.?[0-9]+')
+    postgres_version=${postgres_pkg/postgresql-/}
+    pg_hba="/etc/postgresql/${postgres_version}/main/pg_hba.conf"
+    postgres_password='postgrespostgres'
+
+    sudo sed -i.bak -E 's/^(local\s+all\s+postgres\s+)md5$/\1peer/' "${pg_hba}"
+    sudo service postgresql reload
+
+    password_sql="ALTER USER postgres WITH PASSWORD '${postgres_password}';"
+    sudo -u postgres psql -c "${password_sql}"
+
+    echo "*:*:*:postgres:${postgres_password}" > "${PGPASS}"
+    chmod 600 "${PGPASS}"
+
+    sudo sed -i.bak -E 's/^(local\s+all\s+postgres\s+)peer$/\1md5/' "${pg_hba}"
+    sudo service postgresql reload
+fi