From 74ad6e9bd3a86adc7e105b30127ee956d9b6954a Mon Sep 17 00:00:00 2001 From: Radek Osmulski Date: Wed, 19 Apr 2023 11:56:07 +1000 Subject: [PATCH 1/3] implement review suggestions --- .../transformers-next-item-prediction.ipynb | 80 ++++++++++--------- ...ecase_transformers_next_item_prediction.py | 19 ++++- 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/examples/usecases/transformers-next-item-prediction.ipynb b/examples/usecases/transformers-next-item-prediction.ipynb index 6c0cdffe51..66fe543df5 100644 --- a/examples/usecases/transformers-next-item-prediction.ipynb +++ b/examples/usecases/transformers-next-item-prediction.ipynb @@ -89,7 +89,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-04-17 10:53:46.419725: I tensorflow/core/platform/cpu_feature_guard.cc:194] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: SSE3 SSE4.1 SSE4.2 AVX\n", + "2023-04-19 01:35:02.207465: I tensorflow/core/platform/cpu_feature_guard.cc:194] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: SSE3 SSE4.1 SSE4.2 AVX\n", "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n" ] }, @@ -106,9 +106,9 @@ "text": [ "/usr/local/lib/python3.8/dist-packages/merlin/dtypes/mappings/torch.py:43: UserWarning: PyTorch dtype mappings did not load successfully due to an error: No module named 'torch'\n", " warn(f\"PyTorch dtype mappings did not load successfully due to an error: {exc.msg}\")\n", - "2023-04-17 10:53:47.662982: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-04-17 10:53:47.663430: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-04-17 10:53:47.663612: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n" + "2023-04-19 01:35:03.422817: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2023-04-19 01:35:03.423265: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2023-04-19 01:35:03.423447: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n" ] }, { @@ -125,17 +125,17 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-04-17 10:53:47.910504: I tensorflow/core/platform/cpu_feature_guard.cc:194] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: SSE3 SSE4.1 SSE4.2 AVX\n", + "2023-04-19 01:35:03.695996: I tensorflow/core/platform/cpu_feature_guard.cc:194] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: SSE3 SSE4.1 SSE4.2 AVX\n", "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2023-04-17 10:53:47.911325: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-04-17 10:53:47.911536: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-04-17 10:53:47.911695: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-04-17 10:53:48.671264: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-04-17 10:53:48.671486: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-04-17 10:53:48.671654: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-04-17 10:53:48.671765: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:42] Overriding orig_value setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n", - "2023-04-17 10:53:48.671774: I tensorflow/core/common_runtime/gpu/gpu_process_state.cc:222] Using CUDA malloc Async allocator for GPU: 0\n", - "2023-04-17 10:53:48.671836: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1637] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 24576 MB memory: -> device: 0, name: Quadro RTX 8000, pci bus id: 0000:08:00.0, compute capability: 7.5\n", + "2023-04-19 01:35:03.696824: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2023-04-19 01:35:03.697038: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2023-04-19 01:35:03.697196: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2023-04-19 01:35:04.438845: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2023-04-19 01:35:04.439063: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2023-04-19 01:35:04.439222: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2023-04-19 01:35:04.439332: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:42] Overriding orig_value setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n", + "2023-04-19 01:35:04.439340: I tensorflow/core/common_runtime/gpu/gpu_process_state.cc:222] Using CUDA malloc Async allocator for GPU: 0\n", + "2023-04-19 01:35:04.439400: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1637] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 24576 MB memory: -> device: 0, name: Quadro RTX 8000, pci bus id: 0000:08:00.0, compute capability: 7.5\n", "/usr/local/lib/python3.8/dist-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] @@ -859,7 +859,7 @@ "text": [ "/usr/local/lib/python3.8/dist-packages/keras/initializers/initializers_v2.py:120: UserWarning: The initializer TruncatedNormal is unseeded and being called multiple times, which will return identical values each time (even if the initializer is unseeded). Please update your code to provide a seed to the initializer, or avoid using the same initalizer instance more than once.\n", " warnings.warn(\n", - "2023-04-17 10:53:56.422412: I tensorflow/stream_executor/cuda/cuda_dnn.cc:424] Loaded cuDNN version 8700\n" + "2023-04-19 01:35:13.112516: I tensorflow/stream_executor/cuda/cuda_dnn.cc:424] Loaded cuDNN version 8700\n" ] }, { @@ -893,28 +893,28 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-04-17 10:54:07.156255: W tensorflow/core/grappler/optimizers/loop_optimizer.cc:907] Skipping loop optimization for Merge node with control input: model/xl_net_block/sequential_block_5/replace_masked_embeddings/RaggedWhere/Assert/AssertGuard/branch_executed/_95\n" + "2023-04-19 01:35:23.514177: W tensorflow/core/grappler/optimizers/loop_optimizer.cc:907] Skipping loop optimization for Merge node with control input: model/xl_net_block/sequential_block_5/replace_masked_embeddings/RaggedWhere/Assert/AssertGuard/branch_executed/_95\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2720/2720 [==============================] - 79s 25ms/step - loss: 7.3254 - recall_at_10: 0.1956 - mrr_at_10: 0.0829 - ndcg_at_10: 0.1092 - map_at_10: 0.0829 - precision_at_10: 0.0196 - regularization_loss: 0.0000e+00 - loss_batch: 7.3244\n", + "2720/2720 [==============================] - 77s 24ms/step - loss: 7.3177 - recall_at_10: 0.2005 - mrr_at_10: 0.0877 - ndcg_at_10: 0.1141 - map_at_10: 0.0877 - precision_at_10: 0.0201 - regularization_loss: 0.0000e+00 - loss_batch: 7.3157\n", "Epoch 2/5\n", - "2720/2720 [==============================] - 67s 24ms/step - loss: 6.1170 - recall_at_10: 0.3607 - mrr_at_10: 0.1683 - ndcg_at_10: 0.2137 - map_at_10: 0.1683 - precision_at_10: 0.0361 - regularization_loss: 0.0000e+00 - loss_batch: 6.1153\n", + "2720/2720 [==============================] - 67s 24ms/step - loss: 6.1038 - recall_at_10: 0.3626 - mrr_at_10: 0.1703 - ndcg_at_10: 0.2157 - map_at_10: 0.1703 - precision_at_10: 0.0363 - regularization_loss: 0.0000e+00 - loss_batch: 6.1022\n", "Epoch 3/5\n", - "2720/2720 [==============================] - 68s 25ms/step - loss: 5.5579 - recall_at_10: 0.4363 - mrr_at_10: 0.2080 - ndcg_at_10: 0.2620 - map_at_10: 0.2080 - precision_at_10: 0.0436 - regularization_loss: 0.0000e+00 - loss_batch: 5.5554\n", + "2720/2720 [==============================] - 63s 23ms/step - loss: 5.5767 - recall_at_10: 0.4321 - mrr_at_10: 0.2067 - ndcg_at_10: 0.2600 - map_at_10: 0.2067 - precision_at_10: 0.0432 - regularization_loss: 0.0000e+00 - loss_batch: 5.5751\n", "Epoch 4/5\n", - "2720/2720 [==============================] - 69s 25ms/step - loss: 5.3061 - recall_at_10: 0.4666 - mrr_at_10: 0.2246 - ndcg_at_10: 0.2818 - map_at_10: 0.2246 - precision_at_10: 0.0467 - regularization_loss: 0.0000e+00 - loss_batch: 5.3038\n", + "2720/2720 [==============================] - 64s 23ms/step - loss: 5.3249 - recall_at_10: 0.4631 - mrr_at_10: 0.2223 - ndcg_at_10: 0.2792 - map_at_10: 0.2223 - precision_at_10: 0.0463 - regularization_loss: 0.0000e+00 - loss_batch: 5.3230\n", "Epoch 5/5\n", - "2720/2720 [==============================] - 70s 25ms/step - loss: 5.1669 - recall_at_10: 0.4821 - mrr_at_10: 0.2327 - ndcg_at_10: 0.2917 - map_at_10: 0.2327 - precision_at_10: 0.0482 - regularization_loss: 0.0000e+00 - loss_batch: 5.1647\n" + "2720/2720 [==============================] - 68s 25ms/step - loss: 5.1779 - recall_at_10: 0.4796 - mrr_at_10: 0.2318 - ndcg_at_10: 0.2904 - map_at_10: 0.2318 - precision_at_10: 0.0480 - regularization_loss: 0.0000e+00 - loss_batch: 5.1764\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 15, @@ -961,27 +961,27 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-04-17 10:59:53.461458: W tensorflow/core/grappler/optimizers/loop_optimizer.cc:907] Skipping loop optimization for Merge node with control input: model/xl_net_block/sequential_block_5/replace_masked_embeddings/RaggedWhere/Assert/AssertGuard/branch_executed/_71\n" + "2023-04-19 01:40:55.868086: W tensorflow/core/grappler/optimizers/loop_optimizer.cc:907] Skipping loop optimization for Merge node with control input: model/xl_net_block/sequential_block_5/replace_masked_embeddings/RaggedWhere/Assert/AssertGuard/branch_executed/_71\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "340/340 [==============================] - 11s 20ms/step - loss: 4.7125 - recall_at_10: 0.5556 - mrr_at_10: 0.3126 - ndcg_at_10: 0.3704 - map_at_10: 0.3126 - precision_at_10: 0.0556 - regularization_loss: 0.0000e+00 - loss_batch: 4.7121\n" + "340/340 [==============================] - 10s 18ms/step - loss: 4.7317 - recall_at_10: 0.5562 - mrr_at_10: 0.3114 - ndcg_at_10: 0.3695 - map_at_10: 0.3114 - precision_at_10: 0.0556 - regularization_loss: 0.0000e+00 - loss_batch: 4.7313\n" ] }, { "data": { "text/plain": [ - "{'loss': 4.712502956390381,\n", - " 'recall_at_10': 0.5555964112281799,\n", - " 'mrr_at_10': 0.3125925064086914,\n", - " 'ndcg_at_10': 0.37040114402770996,\n", - " 'map_at_10': 0.3125925064086914,\n", - " 'precision_at_10': 0.05555963143706322,\n", + "{'loss': 4.731719017028809,\n", + " 'recall_at_10': 0.556216835975647,\n", + " 'mrr_at_10': 0.3114089369773865,\n", + " 'ndcg_at_10': 0.36954960227012634,\n", + " 'map_at_10': 0.3114089369773865,\n", + " 'precision_at_10': 0.0556216724216938,\n", " 'regularization_loss': 0.0,\n", - " 'loss_batch': 4.561119079589844}" + " 'loss_batch': 4.579279899597168}" ] }, "execution_count": 16, @@ -1091,15 +1091,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /tmp/tmpvdxq5w0m/model.savedmodel/assets\n" + "INFO:tensorflow:Assets written to: /tmp/tmpk2jcmm5x/model.savedmodel/assets\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /tmp/tmpvdxq5w0m/model.savedmodel/assets\n", - "/usr/local/lib/python3.8/dist-packages/merlin/models/tf/utils/tf_utils.py:100: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n", + "INFO:tensorflow:Assets written to: /tmp/tmpk2jcmm5x/model.savedmodel/assets\n", + "/usr/local/lib/python3.8/dist-packages/merlin/models/tf/utils/tf_utils.py:101: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n", " config[key] = tf.keras.utils.serialize_keras_object(maybe_value)\n", "/usr/local/lib/python3.8/dist-packages/merlin/models/tf/core/combinators.py:288: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n", " config[i] = tf.keras.utils.serialize_keras_object(layer)\n", @@ -1286,7 +1286,7 @@ "output_type": "stream", "text": [ "INFO:tensorflow:Assets written to: /workspace/ensemble/1_predicttensorflowtriton/1/model.savedmodel/assets\n", - "/usr/local/lib/python3.8/dist-packages/merlin/models/tf/utils/tf_utils.py:100: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n", + "/usr/local/lib/python3.8/dist-packages/merlin/models/tf/utils/tf_utils.py:101: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n", " config[key] = tf.keras.utils.serialize_keras_object(maybe_value)\n", "/usr/local/lib/python3.8/dist-packages/merlin/models/tf/core/combinators.py:288: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n", " config[i] = tf.keras.utils.serialize_keras_object(layer)\n", @@ -1397,8 +1397,8 @@ { "data": { "text/plain": [ - "array([[-3.497796 , -3.5179672 , 2.9564662 , ..., -0.45221543,\n", - " -0.8475621 , -1.217337 ]], dtype=float32)" + "array([[-4.958611 , -4.977809 , 0.66012955, ..., -2.32572 ,\n", + " -0.6495375 , -2.4545593 ]], dtype=float32)" ] }, "execution_count": 20, @@ -1428,7 +1428,8 @@ } ], "source": [ - "response.as_numpy('city_id_list/categorical_output').shape" + "predictions = response.as_numpy('city_id_list/categorical_output')\n", + "predictions.shape" ] }, { @@ -1457,7 +1458,8 @@ } ], "source": [ - "wf.output_schema.select_by_name('city_id_list').to_pandas()['properties.embedding_sizes.cardinality'][0]" + "cardinality = wf.output_schema['city_id_list'].properties['embedding_sizes']['cardinality']\n", + "cardinality" ] }, { diff --git a/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py b/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py index e9d5d8646f..08f2646da7 100644 --- a/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py +++ b/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py @@ -33,16 +33,27 @@ def generate_date(): return date df['checkin'] = [generate_date() for _ in range(df.shape[0])] df['checkout'] = [generate_date() for _ in range(df.shape[0])] - df.to_csv('/tmp/train_set.csv') + df.to_csv('/tmpdir/train_set.csv') """ ) tb.cells[4].source = tb.cells[4].source.replace("get_booking('/workspace/data')", "") tb.cells[4].source = tb.cells[4].source.replace( - "read_csv('/workspace/data/train_set.csv'", "read_csv('/tmp/train_set.csv'" + "read_csv('/workspace/data/train_set.csv'", "read_csv('/tmpdir/train_set.csv'" ) tb.cells[31].source = tb.cells[31].source.replace("epochs=5", "epochs=1") - tb.cells[37].source = tb.cells[37].source.replace("/workspace/ensemble", "/tmp/ensemble") + tb.cells[37].source = tb.cells[37].source.replace("/workspace/ensemble", "/tmpdir/ensemble") tb.execute_cell(list(range(0, 38))) - with utils.run_triton_server("/tmp/ensemble", grpc_port=8001): + with run_triton_server("/tmpdir/ensemble", grpc_port=8001): tb.execute_cell(list(range(38, len(tb.cells)))) + + tb.inject( + """ + logits_count = predictions.shape[1] + """ + ) + tb.execute_cell(len(tb.cells) - 1) + + cardinality = tb.ref("cardinality") + logits_count = tb.ref("logits_count") + assert logits_count == cardinality From 6621a8999cd1c7edde6458f3c420a2bb73b4b381 Mon Sep 17 00:00:00 2001 From: Radek Osmulski Date: Wed, 19 Apr 2023 12:12:58 +1000 Subject: [PATCH 2/3] update --- .../test_usecase_transformers_next_item_prediction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py b/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py index 08f2646da7..42f5bcd3b8 100644 --- a/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py +++ b/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py @@ -18,7 +18,7 @@ execute=False, ) @pytest.mark.notebook -def test_next_item_prediction(tb): +def test_next_item_prediction(tb, tmpdir): tb.inject( """ import os, random @@ -44,7 +44,7 @@ def generate_date(): tb.cells[37].source = tb.cells[37].source.replace("/workspace/ensemble", "/tmpdir/ensemble") tb.execute_cell(list(range(0, 38))) - with run_triton_server("/tmpdir/ensemble", grpc_port=8001): + with utils.run_triton_server(f"{tmpdir}/ensemble", grpc_port=8001): tb.execute_cell(list(range(38, len(tb.cells)))) tb.inject( From 5ff0765e7df25d1eb47fc9a3fa3131f969a24de4 Mon Sep 17 00:00:00 2001 From: Radek Osmulski Date: Thu, 20 Apr 2023 09:08:18 +1000 Subject: [PATCH 3/3] fix test --- .../test_usecase_transformers_next_item_prediction.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py b/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py index 42f5bcd3b8..8de1c23ca9 100644 --- a/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py +++ b/tests/unit/tf/examples/test_usecase_transformers_next_item_prediction.py @@ -20,7 +20,7 @@ @pytest.mark.notebook def test_next_item_prediction(tb, tmpdir): tb.inject( - """ + f""" import os, random from datetime import datetime, timedelta from merlin.datasets.synthetic import generate_data @@ -33,15 +33,15 @@ def generate_date(): return date df['checkin'] = [generate_date() for _ in range(df.shape[0])] df['checkout'] = [generate_date() for _ in range(df.shape[0])] - df.to_csv('/tmpdir/train_set.csv') + df.to_csv('{tmpdir}/train_set.csv') """ ) tb.cells[4].source = tb.cells[4].source.replace("get_booking('/workspace/data')", "") tb.cells[4].source = tb.cells[4].source.replace( - "read_csv('/workspace/data/train_set.csv'", "read_csv('/tmpdir/train_set.csv'" + "read_csv('/workspace/data/train_set.csv'", f"read_csv('{tmpdir}/train_set.csv'" ) tb.cells[31].source = tb.cells[31].source.replace("epochs=5", "epochs=1") - tb.cells[37].source = tb.cells[37].source.replace("/workspace/ensemble", "/tmpdir/ensemble") + tb.cells[37].source = tb.cells[37].source.replace("/workspace/ensemble", f"{tmpdir}/ensemble") tb.execute_cell(list(range(0, 38))) with utils.run_triton_server(f"{tmpdir}/ensemble", grpc_port=8001):