Improve Patient Purge Testing

.github/scripts/check-resource-totals-after-patient-purge.sh

@@ -0,0 +1,35 @@
+#!/bin/bash -e
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+. "$SCRIPT_DIR/util.sh"
+
+BASE="http://localhost:8080/fhir"
+ACTUAL_TOTALS="$(curl -sH 'Accept: application/fhir+json' "$BASE/\$totals" | jq -r '.parameter[] | [.name, .valueUnsignedInt] | @csv')"
+EXPECTED_TOTALS="$(cat <<END
+"AllergyIntolerance",2
+"CarePlan",33
+"CareTeam",33
+"Claim",1053
+"Condition",157
+"Device",28
+"DiagnosticReport",1113
+"DocumentReference",407
+"Encounter",407
+"ExplanationOfBenefit",407
+"Immunization",104
+"Location",194
+"Medication",326
+"MedicationAdministration",26
+"MedicationRequest",646
+"Observation",6944
+"Organization",194
+"Patient",8
+"Practitioner",195
+"PractitionerRole",195
+"Procedure",307
+"Provenance",8
+"SupplyDelivery",571
+END
+)"
+
+test "resource totals" "$ACTUAL_TOTALS" "$EXPECTED_TOTALS"

.github/scripts/patient-purge-all.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -e
+
+BASE="http://localhost:8080/fhir"
+
+curl -s "$BASE/Patient?_count=10000&_elements=id" | \
+  jq -r '.entry[].resource.id' | \
+  xargs -P 4 -I {} curl -s -X POST "$BASE/Patient/{}/\$purge" -o /dev/null

.github/workflows/build.yml

@@ -1163,6 +1163,62 @@ jobs:
     - name: Docker Stats
       run: docker stats --no-stream
 
+  integration-test-patient-purge:
+    needs: build
+    runs-on: ubuntu-24.04
+
+    steps:
+    - name: Check out Git repository
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+
+    - name: Install Blazectl
+      run: .github/scripts/install-blazectl.sh
+
+    - name: Download Blaze Image
+      uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+      with:
+        name: blaze-image
+        path: /tmp
+
+    - name: Load Blaze Image
+      run: docker load --input /tmp/blaze.tar
+
+    - name: Run Blaze
+      run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e DB_BLOCK_CACHE_SIZE=512 -e ENABLE_OPERATION_PATIENT_PURGE=true -p 8080:8080 --read-only --tmpfs /tmp:exec -v blaze-data:/app/data blaze:latest
+
+    - name: Wait for Blaze
+      run: .github/scripts/wait-for-url.sh http://localhost:8080/health
+
+    - name: Docker Logs
+      run: docker logs blaze
+
+    - name: Check Capability Statement
+      run: .github/scripts/check-capability-statement.sh
+
+    - name: Check Referential Integrity Enforced
+      run: .github/scripts/check-referential-integrity-enforced.sh
+
+    - name: Load Data
+      run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea
+
+    - name: Check Total-Number of Resources are 92114
+      run: .github/scripts/check-total-number-of-resources.sh 92114
+
+    - name: Patient Purge all
+      run: .github/scripts/patient-purge-all.sh
+
+    - name: Download All Resources
+      run: .github/scripts/download-all-resources.sh
+
+    - name: Download Patient Resources
+      run: .github/scripts/download-resources.sh Patient
+
+    - name: Check Resource Totals
+      run: .github/scripts/check-resource-totals-after-patient-purge.sh
+
+    - name: Docker Stats
+      run: docker stats --no-stream
+
   not-enforcing-referential-integrity-test:
     needs: build
     runs-on: ubuntu-24.04
@@ -1935,6 +1991,7 @@ jobs:
     - cql-expr-cache-test
     - integration-test
     - integration-test-synthea-1000
+    - integration-test-patient-purge
     - not-enforcing-referential-integrity-test
     - admin-api-test
     - small-transactions-test

Makefile

@@ -4,7 +4,7 @@ $(MODULES):
 	$(MAKE) -C $@ $(MAKECMDGOALS)
 
 fmt-root:
-	cljfmt check dev resources src test deps.edn tests.edn
+	cljfmt check dev profiling resources src test deps.edn tests.edn
 
 fmt: $(MODULES) fmt-root
 

docs/implementation/database.md

@@ -36,7 +36,7 @@ Another data model is the document model used in document stores like [MongoDB][
 
 There are two different sets of indices, ones which depend on the database value point in time `t` and the ones which only depend on the content hash of resource versions. The former indices are used to build up the database values were the latter are used to enable queries based on search parameters.
 
-### Indices depending on t
+### Transaction Indices
 
 | Name              | Key Parts | Value                                     |
 |-------------------|-----------|-------------------------------------------|
@@ -118,7 +118,7 @@ The `TypeStats` index keeps track of the total number of resources, and the numb
 
 The `SystemStats` index keeps track of the total number of resources, and the number of changes to all resources at a particular point in time `t`. This statistic is used to populate the total count property in system listings and system history listings in case there are no filters applied.
 
-### Indices not depending on t
+### Search Param Indices
 
 The indices not depending on `t` directly point to the resource versions by their content hash. 
 
@@ -328,6 +328,12 @@ The `delete-history` command is used to delete the history of a resource.
 | type       | yes      | string    | resource type                    |
 | id         | yes      | string    | resource id                      |
 
+#### Execution
+
+* get all instance history entries
+* add the `t` of the transaction as `purged-at?` to the value of each of the history entries not only in the ResourceAsOf index, but also in the TypeAsOf and SystemAsOf index
+* remove the number of history entries purged from the number of changes of `type` and thw whole system
+
 ### Patient Purge
 
 The `patient-purge` command is used to remove all current and historical versions for all resources in a patient compartment.

modules/db/src/blaze/db/api.clj

@@ -490,6 +490,9 @@
   "Returns a CompletableFuture that will complete with the resource of
   `resource-handle` or an anomaly in case of errors.
 
+  Note: If an deleted resource is pulled, a stub with type, id and meta will be
+  returned.
+
   Functions applied after the returned future are executed on the common
   ForkJoinPool."
   [node-or-db resource-handle]

modules/db/src/blaze/db/impl/index/resource_handle.clj

@@ -120,8 +120,8 @@
 (defn reference [rh]
   (str (codec/tid->type (tid rh)) "/" (id rh)))
 
-(defn local-ref-tuple [{type :fhir/type id :id}]
-  [(name type) id])
+(defn local-ref-tuple [rh]
+  [(codec/tid->type (tid rh)) (id rh)])
 
 (defn tid-id [rh]
   (codec/tid-id (tid rh) (codec/id-byte-string (id rh))))

modules/db/src/blaze/db/node/tx_indexer/util.clj

@@ -8,7 +8,7 @@
   {:namespace "blaze"
    :subsystem "db"
    :name "tx_indexer_duration_seconds"}
-  (take 16 (iterate #(* 2 %) 0.00001))
+  (take 16 (iterate #(* 2 %) 0.0001))
   "op")
 
 (defn clauses->query-params [clauses]

modules/db/src/blaze/db/node/tx_indexer/verify.clj

@@ -83,17 +83,21 @@
      refs)))
 
 (def ^:private inc-0 (fnil inc 0))
+(def ^:private dec-0 (fnil dec 0))
 (def ^:private minus-0 (fnil - 0))
 
+(defn- inc-num-changes-and-total [stat]
+  (-> (update stat :num-changes inc-0)
+      (update :total inc-0)))
+
 (defmethod verify "create" [db-before t res {:keys [type id hash refs]}]
   (log/trace (verify-tx-cmd-create-msg type id))
   (with-open [_ (prom/timer tx-u/duration-seconds "verify-create")]
     (check-id-collision! db-before type id)
     (let [tid (codec/tid type)]
       (-> (update res :entries into (index-entries tid id t hash 1 :create refs))
           (update :new-resources conj [type id])
-          (update-in [:stats tid :num-changes] inc-0)
-          (update-in [:stats tid :total] inc-0)))))
+          (update-in [:stats tid] inc-num-changes-and-total)))))
 
 (defn- print-etags [ts]
   (str/join "," (map (partial format "W/\"%d\"") ts)))
@@ -145,6 +149,7 @@
         (and (some? old-t) (= if-none-match old-t))
         (throw-anom (precondition-version-failed-anomaly type id if-none-match))
 
+        ;; identical update, we will do nothing
         (= old-hash hash)
         res
 
@@ -203,7 +208,7 @@
              (update :del-resources conj [type id])
              (update-in [:stats tid :num-changes] inc-0))
           op
-          (update-in [:stats tid :total] (fnil dec 0)))))))
+          (update-in [:stats tid :total] dec-0))))))
 
 (def ^:private ^:const ^long delete-history-max 100000)
 
@@ -219,12 +224,9 @@
 (defn- purge-entry [tid id t rh]
   (rts/index-entries tid id (rh/t rh) (rh/hash rh) (rh/num-changes rh) (rh/op rh) t))
 
-(defn- purge-entries [tid id t instance-history]
-  (into [] (mapcat (partial purge-entry tid id t)) instance-history))
-
-(defn- add-delete-history-entries [entries tid id t instance-history]
-  (-> (update entries :entries into (purge-entries tid (codec/id-byte-string id) t instance-history))
-      (update-in [:stats tid :num-changes] minus-0 (count instance-history))))
+(defn- add-purge-entries [res tid id t resource-handles]
+  (-> (update res :entries into (mapcat (partial purge-entry tid (codec/id-byte-string id) t)) resource-handles)
+      (update-in [:stats tid :num-changes] minus-0 (count resource-handles))))
 
 (defmethod verify "delete-history"
   [db-before t res {:keys [type id]}]
@@ -240,7 +242,7 @@
         (throw-anom (too-many-history-entries-anom type id))
 
         :else
-        (add-delete-history-entries res tid id t instance-history)))))
+        (add-purge-entries res tid id t instance-history)))))
 
 ;; like delete-history but also purges the current version
 (defmethod verify "purge"
@@ -257,10 +259,10 @@
         (throw-anom (too-many-history-entries-anom type id))
 
         :else
-        (cond-> (add-delete-history-entries res tid id t instance-history)
+        (cond-> (add-purge-entries res tid id t instance-history)
           (not (identical? :delete (rh/op (first instance-history))))
           (-> (update :del-resources conj [type id])
-              (update-in [:stats tid :total] (fnil dec 0))))))))
+              (update-in [:stats tid :total] dec-0)))))))
 
 (defmethod verify :default
   [_db-before _t res _tx-cmd]
@@ -336,16 +338,17 @@
 
 (defn- check-referential-integrity!
   [db {:keys [new-resources del-resources]} cmds]
-  (run!
-   (fn [{:keys [op type id refs check-refs]}]
-     (if (#{"delete" "purge"} op)
-       (when check-refs
-         (check-referential-integrity-delete!
-          db del-resources type id))
-       (when refs
-         (check-referential-integrity-write!
-          db new-resources del-resources type id refs))))
-   cmds))
+  (with-open [_ (prom/timer tx-u/duration-seconds "check-referential-integrity")]
+    (run!
+     (fn [{:keys [op type id refs check-refs]}]
+       (if (#{"delete" "purge"} op)
+         (when check-refs
+           (check-referential-integrity-delete!
+            db del-resources type id))
+         (when refs
+           (check-referential-integrity-write!
+            db new-resources del-resources type id refs))))
+     cmds)))
 
 (defn verify-tx-cmds
   "Verifies terminal `tx-cmds`. Returns index entries of the transaction outcome