feat: Add support for running Quartz app enclave in a Docker container (EPID) (#155)

Signed-off-by: Thane Thomson <connect@thanethomson.com>

.dockerignore

@@ -0,0 +1,2 @@
+**/target
+docker

.gitignore

@@ -11,5 +11,6 @@ target/
 artifacts/
 .vscode/
 .DS_Store
+.secrets/
 **/.env.local
 cli/quartz.toml

apps/transfers/enclave/quartz.manifest.template

@@ -1,7 +1,7 @@
-# Quartz manifest file
+# Quartz-based app manifest file
 
 loader.entrypoint = "file:{{ gramine.libos }}"
-libos.entrypoint = "{{ quartz_dir }}/target/release/quartz-app-transfers-enclave"
+libos.entrypoint = "{{ enclave_executable }}"
 
 loader.log_level = "{{ log_level }}"
 
@@ -22,6 +22,7 @@ loader.env.QUARTZ_PORT = { passthrough = true }
 
 loader.argv = ["quartz-app-transfers-enclave",
                 "--chain-id", "testing",
+                "--rpc-addr", "0.0.0.0:11090",
                 "--trusted-height", "{{ trusted_height }}",
                 "--trusted-hash", "{{ trusted_hash }}"]
 
@@ -29,7 +30,7 @@ fs.mounts = [
   { uri = "file:{{ gramine.runtimedir() }}", path = "/lib" },
   { uri = "file:{{ arch_libdir }}", path = "{{ arch_libdir }}" },
   { uri = "file:/usr/{{ arch_libdir }}", path = "/usr{{ arch_libdir }}" },
-  { uri = "file:{{ quartz_dir }}", path = "{{ quartz_dir }}" },
+  { uri = "file:{{ enclave_dir }}", path = "{{ enclave_dir }}" },
 ]
 
 # sgx.debug = true
@@ -43,15 +44,15 @@ sgx.ra_client_linkable = {{ 'true' if ra_client_linkable == '1' else 'false' }}
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",
-  "file:{{ quartz_dir }}/target/release/quartz-app-transfers-enclave",
+  "file:{{ enclave_executable }}",
   "file:{{ gramine.runtimedir() }}/",
   "file:{{ arch_libdir }}/",
   "file:/usr/{{ arch_libdir }}/",
 ]
 
 sgx.allowed_files = [
-  "file:{{ quartz_dir }}/exchange.sk",
-  "file:{{ quartz_dir }}/request.json",
+  "file:{{ enclave_dir }}/exchange.sk",
+  "file:{{ enclave_dir }}/request.json",
 ]
 
 sys.insecure__allow_eventfd = true

apps/transfers/scripts/start.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-#set -eo pipefail
+set -euo pipefail
 
 DIR_QUARTZ=${ROOT:-$(git rev-parse --show-toplevel)}
 DIR_QUARTZ_APP="$DIR_QUARTZ/apps/transfers"
@@ -44,17 +44,18 @@ gramine-sgx-gen-private-key > /dev/null 2>&1 || :  # may fail
 
 echo "... create manifest"
 gramine-manifest  \
--Dlog_level="error"  \
--Dhome="$HOME"  \
--Darch_libdir="/lib/$(gcc -dumpmachine)"  \
--Dra_type="epid" \
--Dra_client_spid="51CAF5A48B450D624AEFE3286D314894" \
--Dra_client_linkable=1 \
--Dquartz_dir="$(pwd)"  \
--Dtrusted_height="$TRUSTED_HEIGHT"  \
--Dtrusted_hash="$TRUSTED_HASH"  \
--Dgramine_port="$QUARTZ_PORT" \
-quartz.manifest.template quartz.manifest
+    -Dlog_level="error"  \
+    -Dhome="${HOME}"  \
+    -Denclave_dir="$(pwd)"  \
+    -Denclave_executable="$(pwd)/target/release/quartz-app-transfers-enclave" \
+    -Darch_libdir="/lib/$(gcc -dumpmachine)"  \
+    -Dra_type="epid" \
+    -Dra_client_spid="51CAF5A48B450D624AEFE3286D314894" \
+    -Dra_client_linkable=1 \
+    -Dtrusted_height="${TRUSTED_HEIGHT}"  \
+    -Dtrusted_hash="${TRUSTED_HASH}"  \
+    -Dgramine_port="${QUARTZ_PORT}" \
+    quartz.manifest.template quartz.manifest
 
 echo "... sign manifest"
 gramine-sgx-sign --manifest quartz.manifest --output quartz.manifest.sgx

docker/README.md

@@ -1,6 +1,7 @@
 # Docker Images
 
-| Image | Description | Link |
-|-------|-------------|------|
-| Single node wasmd testnet | A single node network with a basic wasmd chain setup. Useful for basic, quick testing of your contracts. | [wasmd instructions](./wasmd/README.md) |
-| Single node neutron testnet | A single node neutron network. Useful for testing your contracts on a more advanced cosmwasm chain, with more up to date dependencies. | [neutron instructions](./neutrond/README.md) |
+| Image | Description |
+|-------|-------------|
+| [Single node wasmd testnet](./wasmd/) | A single node network with a basic wasmd chain setup. Useful for basic, quick testing of your contracts. |
+| [Single node neutron testnet](./neutrond/) | A single node neutron network. Useful for testing your contracts on a more advanced cosmwasm chain, with more up to date dependencies. |
+| [SGX enclave](./enclave-sgx/) | A base image for a Quartz enclave to run on an Intel SGX-based machine |

docker/enclave-sgx/Dockerfile

@@ -0,0 +1,65 @@
+FROM rust:1.80-alpine AS build
+
+ARG CARGO_FLAGS=""
+# By default we assume that there is an "enclave" directory in the root of the
+# Quartz app that contains the enclave's code.
+ARG ENCLAVE_DIR="enclave"
+
+COPY . /opt/src
+WORKDIR /opt/src
+
+# TODO: Remove once the Quartz dependencies are open-sourced
+RUN apk update && \
+    apk add --no-cache git openssh && \
+    mkdir -m 0700 /root/.ssh && \
+    cp .secrets/* /root/.ssh/ && \
+    chmod 0600 /root/.ssh/* && \
+    chmod 0644 /root/.ssh/*.pub && \
+    ssh-keyscan github.com >> /root/.ssh/known_hosts
+
+# System dependencies for building our binary
+RUN apk update && \
+    apk add --no-cache build-base protobuf-dev
+
+RUN cd /opt/src/${ENCLAVE_DIR} && \
+    CARGO_TARGET_DIR=./target cargo build --release ${CARGO_FLAGS}
+
+# TODO: Remove once the Quartz dependencies are open-sourced
+RUN rm -rf /root/.ssh/
+
+#------------------------------------------------------------------------------
+
+FROM gramineproject/gramine:1.7-jammy
+
+ARG ENCLAVE_DIR="enclave"
+# By default we assume that the enclave binary's name is just "enclave".
+ARG ENCLAVE_BIN="enclave"
+ARG TRUSTED_HEIGHT
+ARG TRUSTED_HASH
+
+RUN apt update && \
+    apt install -y build-essential
+
+# Copy the enclave binary we built in the previous stage
+COPY --from=build /opt/src/${ENCLAVE_DIR}/target/release/${ENCLAVE_BIN} /opt/enclave/bin/enclave
+COPY --from=build /opt/src/${ENCLAVE_DIR}/quartz.manifest.template /opt/enclave/
+
+WORKDIR /opt/enclave
+
+RUN gramine-sgx-gen-private-key > /dev/null 2>&1 && \
+    gramine-manifest  \
+        -Dlog_level="error"  \
+        -Dhome="/opt/enclave"  \
+        -Denclave_dir="/opt/enclave"  \
+        -Denclave_executable="/opt/enclave/bin/enclave" \
+        -Darch_libdir="/lib/$(gcc -dumpmachine)"  \
+        -Dra_type="epid" \
+        -Dra_client_spid="51CAF5A48B450D624AEFE3286D314894" \
+        -Dra_client_linkable=1 \
+        -Dtrusted_height="${TRUSTED_HEIGHT}"  \
+        -Dtrusted_hash="${TRUSTED_HASH}"  \
+        -Dgramine_port=11090 \
+        quartz.manifest.template quartz.manifest && \
+    gramine-sgx-sign --manifest quartz.manifest --output quartz.manifest.sgx
+
+CMD ["/restart_aesm.sh && gramine-sgx ./quartz"]

docker/enclave-sgx/README.md

@@ -0,0 +1,114 @@
+# Quartz Enclave Build/Run Image for SGX
+
+This folder contains the basis for a multi-stage Docker image that:
+
+1. Builds the enclave
+2. Takes the binary from the build stage and embeds it in the [Gramine Docker
+   image][gramine-docker], such that it can run on an SGX-enabled machine.
+
+## Requirements
+
+- Docker
+
+The build process itself does not require an SGX-capable processor, but running
+the image does.
+
+## Setup
+
+### Secrets
+
+**TODO: Remove this subsection once all necessary subcomponents are
+open-sourced.**
+
+Before building the image, you will need to ensure that the image has access to
+a public/private keypair that will allow access to the private dependencies
+needed by the build process.
+
+For example, you could generate a public/private keypair as follows. **NB: This
+keypair must _not_ be password-protected, since it needs to be accessible in an
+unsupervised manner during the Docker image build.**
+
+```bash
+# Should generate ~/.ssh/{id_ed25519,id_ed25519.pub}
+ssh-keygen -t ed25519
+```
+
+Both the public and private keys must be copied into a `.secrets` folder in the
+root of this repository prior to building the image.
+
+```bash
+# From the root of the cycles-quartz repo
+mkdir -p .secrets
+cp ~/.ssh/id_ed25519* .secrets/
+```
+
+You will, of course, need to make sure that you have added the public key to
+your GitHub account in your SSH key settings before these keys will be useful.
+Once you have built the image, you can delete the keys and remove them from your
+GitHub account.
+
+### Trusted Height and Hash
+
+The enclave needs to know that it can trust the chain with which it interacts,
+and to do so it uses a light client that needs to be initialized with a root
+trusted height along with the hash of the block at that height.
+
+A simple way of initializing such a light client for testing/experimentation
+purposes is to query the chain:
+
+```bash
+# Assumes a CometBFT-based chain accessible via localhost. Replace "localhost"
+# with the host/IP address of the full node/validator you want to query.
+#
+# Gets the latest block for the chain. Pipes the output through jq to format it
+# nicely.
+curl http://localhost:26657/block | jq
+```
+
+The hash in which we are interested is the _last_ block hash, meaning that the
+height of that block is the height of the response from the above command minus
+1.
+
+## Building
+
+As an example, to build a Docker image for the transfers app's enclave:
+
+```bash
+# From the root of the cycles-quartz repository
+docker build \
+   --platform linux/amd64 \
+   --build-arg ENCLAVE_DIR=apps/transfers/enclave \
+   --build-arg ENCLAVE_BIN=quartz-app-transfers-enclave \
+   --build-arg TRUSTED_HEIGHT=1234 \
+   --build-arg TRUSTED_HASH=0123456789abcdef \
+   -t informaldev/transfers-enclave \
+   -f ./docker/enclave-sgx/Dockerfile \
+   .
+```
+
+This builds an image tagged `informaldev/transfers-enclave:latest`.
+
+The following build arguments are important:
+
+- `ENCLAVE_DIR` - The relative path, from the root of the repository, to the
+  enclave source code.
+- `ENCLAVE_BIN` - The filename of the enclave binary (usually defined in the
+  `Cargo.toml` file).
+- `TRUSTED_HEIGHT` - The trusted height of the chain to build into the image for
+  the enclave light client.
+- `TRUSTED_HASH` - The trusted hash of the chain to build into the image for the
+  enclave light client.
+
+## Running
+
+On an SGX-enabled machine:
+
+```bash
+# The devices need to be mounted into the container.
+docker run --rm -it \
+   --device /dev/sgx_enclave \
+   --device /dev/sgx_provision \
+   informaldev/transfers-enclave
+```
+
+[gramine-docker]: https://hub.docker.com/r/gramineproject/gramine

relayer/scripts/relay.sh

@@ -58,7 +58,10 @@ rm -f "$QUOTE_FILE" "$REPORT_FILE" "$REPORT_SIG_FILE"
 
 # request the IAS report for EPID attestations
 echo -n "$QUOTE" | xxd -r -p - > "$QUOTE_FILE"
-gramine-sgx-ias-request report -g "$RA_CLIENT_SPID" -k "$IAS_API_KEY" -q "$QUOTE_FILE" -r "$REPORT_FILE" -s "$REPORT_SIG_FILE" > /dev/null 2>&1
+docker run --rm -it \
+    -v /tmp:/tmp:rw \
+    gramineproject/gramine:1.7-jammy \
+    "gramine-sgx-ias-request report -g \"$RA_CLIENT_SPID\" -k \"$IAS_API_KEY\" -q \"$QUOTE_FILE\" -r \"$REPORT_FILE\" -s \"$REPORT_SIG_FILE\" > /dev/null 2>&1"
 REPORT=$(cat "$REPORT_FILE")
 REPORTSIG=$(cat "$REPORT_SIG_FILE" | tr -d '\r')