feat: add migration flavor to hermetic driver (#152)

Co-authored-by: Mohsin Zaidi <2236875+smrz2001@users.noreply.github.com>

.github/workflows/hermetic.yml

@@ -135,12 +135,46 @@ jobs:
               test_selector="fast"
           fi
           make TEST_SELECTOR="$test_selector" HERMETIC_CMD=./bin/hermetic-driver hermetic-tests
+  run-migration-tests:
+    name: Test Migrations
+    runs-on: ubuntu-latest
+    environment: test
+    needs:
+      - build-driver
+      - publish-suite
+      - generate-matrix #Needed to know the BUILD_TAG
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Setup GKE auth
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: ${{ secrets.GKE_SA_KEY }}
+      -
+        name: Get GKE credentials
+        uses: 'google-github-actions/get-gke-credentials@v1'
+        with:
+          cluster_name: ${{ vars.GKE_CLUSTER }}
+          location: ${{ vars.GKE_ZONE }}
+      - uses: actions/download-artifact@master
+        with:
+          name: hermetic-driver
+          path: ./bin
+      -
+        name: Test ${{ matrix.networks }}
+        run: |
+          set -euxo pipefail
+          export BUILD_TAG=${{ needs.generate-matrix.outputs.build_tag }}
+          chmod +x ./bin/hermetic-driver
+          make HERMETIC_CMD=./bin/hermetic-driver migration-tests
 
   collect-results:
     name: Hermetic Test Results
     if: ${{ always() }}
     runs-on: ubuntu-latest
-    needs: [run-tests]
+    needs: [run-tests, run-migration-tests]
     steps:
       - run: exit 1
         # see https://stackoverflow.com/a/67532120/4907315

Makefile

@@ -10,6 +10,10 @@ BUILD_PROFILE ?= release
 BUILD_TAG ?= dev-run
 # Path to network to test against.
 TEST_NETWORK ?= ./networks/basic-rust.yaml
+# Path to migration network to test against.
+TEST_PRE_MIGRATION_NETWORK ?= ./migration-networks/basic-go-rust-pre.yaml
+# Path to migration network to test against.
+TEST_POST_MIGRATION_NETWORK ?= ./migration-networks/basic-go-rust-post.yaml
 # Path to performance simulation.
 TEST_SIMULATION ?= ./simulations/basic-simulation.yaml
 # Name for the test suite image, without any tag
@@ -104,6 +108,18 @@ hermetic-tests:
 		--test-image "${TEST_SUITE_IMAGE}" \
 		--test-selector "${TEST_SELECTOR}"
 
+.PHONY: migration-tests
+migration-tests:
+	${HERMETIC_CMD} test \
+		--network "${TEST_PRE_MIGRATION_NETWORK}" \
+		--flavor migration \
+		--suffix "${HERMETIC_SUFFIX}" \
+		--network-ttl ${HERMETIC_TTL} \
+		--test-image "${TEST_SUITE_IMAGE}" \
+		--test-selector "migration" \
+		--migration-wait-secs 400 \
+		--migration-network "${TEST_POST_MIGRATION_NETWORK}"
+
 .PHONY: performance-tests
 performance-tests:
 	${HERMETIC_CMD} test \

README.md

@@ -29,13 +29,18 @@ Testing against durable infrastructure allows for better coverage of real world
 There are several test flavors:
 
 * Correctness tests
+* Migration tests
 * Performance tests
 
 More will be added as needed.
 
 The correctness tests, test a specific property of a network (i.e. writes can be read).
 These tests do not assume any network topology.
-Property tests live in this repo in the `/suite` directory.
+Correctness tests live in this repo in the `/suite` directory.
+
+The migration tests run tests specific to migrating from older versions to newer versions of Ceramic.
+These tests assume that when the version of the Ceramic process changes the migration is complete and the test can continue to validate the migration.
+Migrations tests live in this repo in the `/suite/src/__tests__/migration` directory.
 
 The performance tests, test at scale.
 These tests do not assume any network topology.

hermetic/src/cli.rs

@@ -60,12 +60,25 @@ pub struct TestOpts {
     /// Path regex passed to Jest to select which tests to run.
     #[arg(long, default_value = ".")]
     test_selector: String,
+
+    /// Path to migration network yaml file.
+    /// Required with flavor is `migration`.
+    #[arg(long)]
+    migration_network: Option<PathBuf>,
+
+    /// Number of seconds to wait after starting the test job before starting the migration
+    /// network.
+    /// Required with flavor is `migration`.
+    #[arg(long)]
+    migration_wait_secs: Option<u64>,
 }
 
 #[derive(Debug, Clone)]
 pub enum FlavorOpts {
     /// Correctness tests
     Correctness,
+    /// Migration tests
+    Migration,
     /// Performance tests
     Performance,
 }
@@ -74,14 +87,19 @@ impl FlavorOpts {
     fn name(&self) -> &'static str {
         match self {
             FlavorOpts::Correctness => "correctness",
+            FlavorOpts::Migration => "migration",
             FlavorOpts::Performance => "perf",
         }
     }
 }
 
 impl ValueEnum for FlavorOpts {
     fn value_variants<'a>() -> &'a [Self] {
-        &[FlavorOpts::Correctness, FlavorOpts::Performance]
+        &[
+            FlavorOpts::Correctness,
+            FlavorOpts::Migration,
+            FlavorOpts::Performance,
+        ]
     }
 
     fn to_possible_value(&self) -> Option<clap::builder::PossibleValue> {
@@ -109,10 +127,18 @@ impl TryFrom<TestOpts> for TestConfig {
             network_timeout,
             job_timeout,
             test_selector,
+            migration_network,
+            migration_wait_secs,
         } = opts;
 
         let flavor = match flavor {
             FlavorOpts::Correctness => Flavor::Correctness,
+            FlavorOpts::Migration => Flavor::Migration {
+                wait_secs: migration_wait_secs
+                    .ok_or(anyhow!("Migration flavor requires `migration_wait_secs`"))?,
+                migration: migration_network
+                    .ok_or(anyhow!("Migration flavor requires `migration_network`"))?,
+            },
             FlavorOpts::Performance => Flavor::Performance(
                 simulation.ok_or(anyhow!("Simulation file required for performance tests"))?,
             ),

hermetic/src/cli/tester.rs

@@ -1,4 +1,9 @@
-use std::{collections::BTreeMap, fmt::Display, path::PathBuf};
+use std::{
+    collections::BTreeMap,
+    fmt::Display,
+    path::{Path, PathBuf},
+    time::Duration,
+};
 
 use anyhow::{anyhow, Result};
 use futures::{future::BoxFuture, StreamExt, TryStreamExt};
@@ -32,7 +37,7 @@ use kube::{
 };
 use log::{debug, info, trace};
 use serde::{de::DeserializeOwned, Serialize};
-use tokio::fs;
+use tokio::{fs, time::sleep};
 
 const TESTER_NAME: &str = "ceramic-tester";
 const CERAMIC_ADMIN_DID_SECRET_NAME: &str = "ceramic-admin";
@@ -73,6 +78,8 @@ pub struct TestConfig {
 pub enum Flavor {
     /// Correctness tests
     Correctness,
+    /// Migration tests
+    Migration { wait_secs: u64, migration: PathBuf },
     /// Performance tests
     Performance(PathBuf),
 }
@@ -81,6 +88,7 @@ impl Flavor {
     fn name(&self) -> &'static str {
         match self {
             Flavor::Correctness => "correctness",
+            Flavor::Migration { .. } => "migration",
             Flavor::Performance(_) => "perf",
         }
     }
@@ -92,34 +100,47 @@ impl Display for Flavor {
     }
 }
 
-pub async fn run(opts: TestConfig) -> Result<()> {
-    // Infer the runtime environment and try to create a Kubernetes Client
-    let client = Client::try_default().await?;
-
+async fn parse_network_file(
+    file_path: impl AsRef<Path>,
+    ttl: u64,
+    flavor: &Flavor,
+    suffix: &Option<String>,
+) -> Result<Network> {
     // Parse network file
-    let mut network: Network = serde_yaml::from_str(&fs::read_to_string(&opts.network).await?)?;
+    let mut network: Network = serde_yaml::from_str(&fs::read_to_string(file_path).await?)?;
     debug!("input network {:#?}", network);
 
     // The test driver relies on the Keramik operator network TTL to clean up the network, with an 8 hour default that
     // allows devs to investigate any failures. The TTL can also be extended at any time, if more time is needed.
-    network.spec.ttl_seconds = Some(opts.network_ttl);
+    network.spec.ttl_seconds = Some(ttl);
 
     let mut network_name = format!(
         "{}-{}",
-        opts.flavor,
+        flavor,
         network
             .metadata
             .name
             .as_ref()
             .expect("network should have a defined name in metadata")
     );
-    if let Some(suffix) = &opts.suffix {
+    if let Some(suffix) = &suffix {
         if !suffix.is_empty() {
             network_name = format!("{network_name}-{suffix}");
         }
     }
     network.metadata.name = Some(network_name.clone());
+    Ok(network)
+}
+
+pub async fn run(opts: TestConfig) -> Result<()> {
+    // Infer the runtime environment and try to create a Kubernetes Client
+    let client = Client::try_default().await?;
+
+    // Parse network file
+    let network =
+        parse_network_file(opts.network, opts.network_ttl, &opts.flavor, &opts.suffix).await?;
     debug!("configured network {:#?}", network);
+    let network_name = network.name_unchecked();
 
     let namespace = format!("keramik-{network_name}");
 
@@ -166,7 +187,7 @@ pub async fn run(opts: TestConfig) -> Result<()> {
 
     // Create any dependencies of the job
     match opts.flavor {
-        Flavor::Performance(_) => {}
+        Flavor::Performance(_) | Flavor::Migration { .. } => {}
         Flavor::Correctness => {
             apply_resource_namespaced(
                 client.clone(),
@@ -184,7 +205,7 @@ pub async fn run(opts: TestConfig) -> Result<()> {
 
     // Create the job/simulation
     let job_name = match &opts.flavor {
-        Flavor::Correctness => {
+        Flavor::Correctness | Flavor::Migration { .. } => {
             create_resource_namespaced(
                 client.clone(),
                 &namespace,
@@ -206,6 +227,21 @@ pub async fn run(opts: TestConfig) -> Result<()> {
         Flavor::Correctness => {
             wait_for_job(client.clone(), &namespace, &job_name, opts.job_timeout).await?
         }
+        Flavor::Migration {
+            wait_secs,
+            ref migration,
+        } => {
+            // Wait designated time before starting the migration
+            sleep(Duration::from_secs(wait_secs)).await;
+            let migration_network =
+                parse_network_file(migration, opts.network_ttl, &opts.flavor, &opts.suffix).await?;
+
+            // Apply the migration network
+            apply_resource(client.clone(), migration_network).await?;
+
+            // Finally wait for the test job to finish
+            wait_for_job(client.clone(), &namespace, &job_name, opts.job_timeout).await?
+        }
         Flavor::Performance(_) => {
             wait_for_simulation(client.clone(), &namespace, opts.job_timeout).await?
         }

migration-networks/basic-go-rust-post.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: "keramik.3box.io/v1alpha1"
+kind: Network
+metadata:
+  name: basic-go-rust
+spec:
+  replicas: 1
+  ceramic:
+    - env:
+        CERAMIC_RECON_MODE: "true"
+      ipfs:
+        rust:
+          env:
+            CERAMIC_ONE_RECON: "true"
+          resourceLimits:
+            cpu: "4"
+            memory: "1Gi"
+          migrationCmd:
+            - "from-ipfs"
+            - "-i"
+            - "/data/ipfs/blocks"
+            - "-o"
+            - "/data/ipfs/"
+            - "--network"
+            - "dev-unstable"
+  # Use Kubo with CAS because it still needs pubsub
+  cas:
+    casResourceLimits:
+      cpu: "2"
+      memory: "4Gi"
+    ipfs:
+      go: {}

migration-networks/basic-go-rust-pre.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: "keramik.3box.io/v1alpha1"
+kind: Network
+metadata:
+  name: basic-go-rust
+spec:
+  replicas: 1
+  ceramic:
+    - ipfs:
+        go:
+          resourceLimits:
+            cpu: "4"
+            memory: "1Gi"
+  # Use Kubo with CAS because it still needs pubsub
+  cas:
+    casResourceLimits:
+      cpu: "2"
+      memory: "4Gi"
+    ipfs:
+      go: {}

suite/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "scripts": {
     "build": "tsc",
-    "test": "NODE_OPTIONS=--experimental-vm-modules jest --runInBand --reporters default --setupFiles dotenv/config",
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest --runInBand --reporters default --setupFiles dotenv/config --forceExit",
     "format": "prettier --write '**/*.{js,ts,jsx,tsx,json,css,scss,md}'"
   },
   "devDependencies": {