Update testrunner to use new dev-env [WIP] (#1575)

* Update testrunner to use new dev-env

* Fix label testcase

* Vendor the dev-infra scripts from the atproto repo for the dev-env server runner

* Bump detox to fix the ios sim control issue

* Use iphone 15 pro for tests

* Ensure the reminders never trigger during tests

* Skip the shell tests due to a crash bug with detox and the drawer

5 files changed, 254 insertions, 60 deletions

diff --git a/jest/dev-infra/_common.sh b/jest/dev-infra/_common.sh
new file mode 100755
index 000000000..0d66653c8
--- /dev/null
+++ b/jest/dev-infra/_common.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env sh
+
+get_container_id() {
+  local compose_file=$1
+  local service=$2
+  if [ -z "${compose_file}" ] || [ -z "${service}" ]; then
+    echo "usage: get_container_id <compose_file> <service>"
+    exit 1
+  fi
+
+  docker compose -f $compose_file ps --format json --status running \
+    | jq -r '.[]? | select(.Service == "'${service}'") | .ID'
+}
+
+# Exports all environment variables
+export_env() {
+  export_pg_env
+  export_redis_env
+}
+
+# Exports postgres environment variables
+export_pg_env() {
+  # Based on creds in compose.yaml
+  export PGPORT=5433
+  export PGHOST=localhost
+  export PGUSER=pg
+  export PGPASSWORD=password
+  export PGDATABASE=postgres
+  export DB_POSTGRES_URL="postgresql://pg:password@127.0.0.1:5433/postgres"
+}
+
+# Exports redis environment variables
+export_redis_env() {
+  export REDIS_HOST="127.0.0.1:6380"
+}
+
+# Main entry point
+main() {
+  # Expect a SERVICES env var to be set with the docker service names
+  local services=${SERVICES}
+
+  dir=$(dirname $0)
+  compose_file="${dir}/docker-compose.yaml"
+
+  # whether this particular script started the container(s)
+  started_container=false
+
+  # trap SIGINT and performs cleanup as necessary, i.e.
+  # taking down containers if this script started them
+  trap "on_sigint ${services}" INT
+  on_sigint() {
+    local services=$@
+    echo # newline
+    if $started_container; then
+      docker compose -f $compose_file rm -f --stop --volumes ${services}
+    fi
+    exit $?
+  }
+
+  # check if all services are running already
+  not_running=false
+  for service in $services; do
+    container_id=$(get_container_id $compose_file $service)
+    if [ -z $container_id ]; then
+      not_running=true
+      break
+    fi
+  done
+
+  # if any are missing, recreate all services
+  if $not_running; then
+    docker compose -f $compose_file up --wait --force-recreate ${services}
+    started_container=true
+  else
+    echo "all services ${services} are already running"
+  fi
+
+  # setup environment variables and run args
+  export_env
+  "$@"
+  # save return code for later
+  code=$?
+
+  # performs cleanup as necessary, i.e. taking down containers
+  # if this script started them
+  echo # newline
+  if $started_container; then
+    docker compose -f $compose_file rm -f --stop --volumes ${services}
+  fi
+
+  exit ${code}
+}
diff --git a/jest/dev-infra/docker-compose.yaml b/jest/dev-infra/docker-compose.yaml
new file mode 100644
index 000000000..3d582c18b
--- /dev/null
+++ b/jest/dev-infra/docker-compose.yaml
@@ -0,0 +1,49 @@
+version: '3.8'
+services:
+  # An ephermerally-stored postgres database for single-use test runs
+  db_test: &db_test
+    image: postgres:14.4-alpine
+    environment:
+      - POSTGRES_USER=pg
+      - POSTGRES_PASSWORD=password
+    ports:
+      - '5433:5432'
+    # Healthcheck ensures db is queryable when `docker-compose up --wait` completes
+    healthcheck:
+      test: 'pg_isready -U pg'
+      interval: 500ms
+      timeout: 10s
+      retries: 20
+  # A persistently-stored postgres database
+  db:
+    <<: *db_test
+    ports:
+      - '5432:5432'
+    healthcheck:
+      disable: true
+    volumes:
+      - atp_db:/var/lib/postgresql/data
+  # An ephermerally-stored redis cache for single-use test runs
+  redis_test: &redis_test
+    image: redis:7.0-alpine
+    ports:
+      - '6380:6379'
+    # Healthcheck ensures redis is queryable when `docker-compose up --wait` completes
+    healthcheck:
+      test: ['CMD-SHELL', '[ "$$(redis-cli ping)" = "PONG" ]']
+      interval: 500ms
+      timeout: 10s
+      retries: 20
+  # A persistently-stored redis cache
+  redis:
+    <<: *redis_test
+    command: redis-server --save 60 1 --loglevel warning
+    ports:
+      - '6379:6379'
+    healthcheck:
+      disable: true
+    volumes:
+      - atp_redis:/data
+volumes:
+  atp_db:
+  atp_redis:
diff --git a/jest/dev-infra/with-test-db.sh b/jest/dev-infra/with-test-db.sh
new file mode 100755
index 000000000..cc083491a
--- /dev/null
+++ b/jest/dev-infra/with-test-db.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+# Example usage:
+# ./with-test-db.sh psql postgresql://pg:password@localhost:5433/postgres -c 'select 1;'
+
+dir=$(dirname $0)
+. ${dir}/_common.sh
+
+SERVICES="db_test" main "$@"
diff --git a/jest/dev-infra/with-test-redis-and-db.sh b/jest/dev-infra/with-test-redis-and-db.sh
new file mode 100755
index 000000000..c2b0c75ff
--- /dev/null
+++ b/jest/dev-infra/with-test-redis-and-db.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+
+# Example usage:
+# ./with-test-redis-and-db.sh psql postgresql://pg:password@localhost:5433/postgres -c 'select 1;'
+# ./with-test-redis-and-db.sh redis-cli -h localhost -p 6380 ping
+
+dir=$(dirname $0)
+. ${dir}/_common.sh
+
+SERVICES="db_test redis_test" main "$@"
diff --git a/jest/test-pds.ts b/jest/test-pds.ts
index 37ad824a0..bc3692600 100644
--- a/jest/test-pds.ts
+++ b/jest/test-pds.ts
@@ -1,7 +1,7 @@
 import net from 'net'
 import path from 'path'
 import fs from 'fs'
-import {TestNetworkNoAppView} from '@atproto/dev-env'
+import {TestNetwork} from '@atproto/dev-env'
 import {AtUri, BskyAgent} from '@atproto/api'
 
 export interface TestUser {
@@ -18,14 +18,59 @@ export interface TestPDS {
   close: () => Promise<void>
 }
 
+class StringIdGenerator {
+  _nextId = [0]
+  constructor(
+    public _chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ',
+  ) {}
+
+  next() {
+    const r = []
+    for (const char of this._nextId) {
+      r.unshift(this._chars[char])
+    }
+    this._increment()
+    return r.join('')
+  }
+
+  _increment() {
+    for (let i = 0; i < this._nextId.length; i++) {
+      const val = ++this._nextId[i]
+      if (val >= this._chars.length) {
+        this._nextId[i] = 0
+      } else {
+        return
+      }
+    }
+    this._nextId.push(0)
+  }
+
+  *[Symbol.iterator]() {
+    while (true) {
+      yield this.next()
+    }
+  }
+}
+
+const ids = new StringIdGenerator()
+
 export async function createServer(
   {inviteRequired}: {inviteRequired: boolean} = {inviteRequired: false},
 ): Promise<TestPDS> {
   const port = await getPort()
   const port2 = await getPort(port + 1)
   const pdsUrl = `http://localhost:${port}`
-  const testNet = await TestNetworkNoAppView.create({
-    pds: {port, publicUrl: pdsUrl, inviteRequired},
+  const id = ids.next()
+  const testNet = await TestNetwork.create({
+    pds: {
+      port,
+      publicUrl: pdsUrl,
+      inviteRequired,
+      dbPostgresSchema: `pds_${id}`,
+    },
+    bsky: {
+      dbPostgresSchema: `bsky_${id}`,
+    },
     plc: {port: port2},
   })
 
@@ -48,7 +93,7 @@ class Mocker {
   users: Record<string, TestUser> = {}
 
   constructor(
-    public testNet: TestNetworkNoAppView,
+    public testNet: TestNetwork,
     public service: string,
     public pic: Uint8Array,
   ) {
@@ -59,6 +104,10 @@ class Mocker {
     return this.testNet.pds
   }
 
+  get bsky() {
+    return this.testNet.bsky
+  }
+
   get plc() {
     return this.testNet.plc
   }
@@ -81,11 +130,7 @@ class Mocker {
     const inviteRes = await agent.api.com.atproto.server.createInviteCode(
       {useCount: 1},
       {
-        headers: {
-          authorization: `Basic ${btoa(
-            `admin:${this.pds.ctx.cfg.adminPassword}`,
-          )}`,
-        },
+        headers: this.pds.adminAuthHeaders('admin'),
         encoding: 'application/json',
       },
     )
@@ -260,11 +305,7 @@ class Mocker {
     await agent.api.com.atproto.server.createInviteCode(
       {useCount: 1, forAccount},
       {
-        headers: {
-          authorization: `Basic ${btoa(
-            `admin:${this.pds.ctx.cfg.adminPassword}`,
-          )}`,
-        },
+        headers: this.pds.adminAuthHeaders('admin'),
         encoding: 'application/json',
       },
     )
@@ -275,24 +316,21 @@ class Mocker {
     if (!did) {
       throw new Error(`Invalid user: ${user}`)
     }
-    const ctx = this.pds.ctx
+    const ctx = this.bsky.ctx
     if (!ctx) {
-      throw new Error('Invalid PDS')
+      throw new Error('Invalid appview')
     }
-
-    await ctx.db.db
-      .insertInto('label')
-      .values([
-        {
-          src: ctx.cfg.labelerDid,
-          uri: did,
-          cid: '',
-          val: label,
-          neg: 0,
-          cts: new Date().toISOString(),
-        },
-      ])
-      .execute()
+    const labelSrvc = ctx.services.label(ctx.db.getPrimary())
+    await labelSrvc.createLabels([
+      {
+        src: ctx.cfg.labelerDid,
+        uri: did,
+        cid: '',
+        val: label,
+        neg: false,
+        cts: new Date().toISOString(),
+      },
+    ])
   }
 
   async labelProfile(label: string, user: string) {
@@ -307,43 +345,39 @@ class Mocker {
       rkey: 'self',
     })
 
-    const ctx = this.pds.ctx
+    const ctx = this.bsky.ctx
     if (!ctx) {
-      throw new Error('Invalid PDS')
+      throw new Error('Invalid appview')
     }
-    await ctx.db.db
-      .insertInto('label')
-      .values([
-        {
-          src: ctx.cfg.labelerDid,
-          uri: profile.uri,
-          cid: profile.cid,
-          val: label,
-          neg: 0,
-          cts: new Date().toISOString(),
-        },
-      ])
-      .execute()
+    const labelSrvc = ctx.services.label(ctx.db.getPrimary())
+    await labelSrvc.createLabels([
+      {
+        src: ctx.cfg.labelerDid,
+        uri: profile.uri,
+        cid: profile.cid,
+        val: label,
+        neg: false,
+        cts: new Date().toISOString(),
+      },
+    ])
   }
 
   async labelPost(label: string, {uri, cid}: {uri: string; cid: string}) {
-    const ctx = this.pds.ctx
+    const ctx = this.bsky.ctx
     if (!ctx) {
-      throw new Error('Invalid PDS')
+      throw new Error('Invalid appview')
     }
-    await ctx.db.db
-      .insertInto('label')
-      .values([
-        {
-          src: ctx.cfg.labelerDid,
-          uri,
-          cid,
-          val: label,
-          neg: 0,
-          cts: new Date().toISOString(),
-        },
-      ])
-      .execute()
+    const labelSrvc = ctx.services.label(ctx.db.getPrimary())
+    await labelSrvc.createLabels([
+      {
+        src: ctx.cfg.labelerDid,
+        uri,
+        cid,
+        val: label,
+        neg: false,
+        cts: new Date().toISOString(),
+      },
+    ])
   }
 
   async createMuteList(user: string, name: string): Promise<string> {