diff --git a/.gitattributes b/.gitattributes index cc8636cc1d..c711b36c9a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7,3 +7,5 @@ tests/mock_triton_server/payloads/** filter=lfs diff=lfs merge=lfs -text tests/tests_data/** filter=lfs diff=lfs merge=lfs -text examples/basic_usage/img/** filter=lfs diff=lfs merge=lfs -text docs/source/img/* filter=lfs diff=lfs merge=lfs -text +git filter=lfs diff=lfs merge=lfs -text +status filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index bdd48c234e..eb791243c1 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -46,7 +46,7 @@ jobs: uses: ./.github/workflows/ci_pipe.yml with: run_check: ${{ startsWith(github.ref_name, 'pull-request/') }} - container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-230801 - test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-230801 + container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-230828 + test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-230828 secrets: NGC_API_KEY: ${{ secrets.NGC_API_KEY }} diff --git a/ci/runner/Dockerfile b/ci/runner/Dockerfile index 43a4b120b2..7942224235 100644 --- a/ci/runner/Dockerfile +++ b/ci/runner/Dockerfile @@ -66,13 +66,6 @@ RUN apt update && \ COPY ./docker/conda/environments/cuda${CUDA_SHORT_VER}_examples.yml /tmp/conda/cuda${CUDA_SHORT_VER}_examples.yml COPY ./ci/scripts/download_kafka.py /tmp/scripts/download_kafka.py -# Install extra deps needed for gnn_fraud_detection_pipeline & ransomware_detection examples -RUN CONDA_ALWAYS_YES=true /opt/conda/bin/mamba env update -n ${PROJ_NAME} -q --file /tmp/conda/cuda${CUDA_SHORT_VER}_examples.yml && \ - conda clean -afy && \ - source activate ${PROJ_NAME} && \ - pip install --ignore-requires-python stellargraph==1.2.1 && \ - rm -rf /tmp/conda - # Install camouflage needed for unittests to mock a triton server RUN source activate ${PROJ_NAME} && \ npm install -g camouflage-server@0.9 && \ diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh index 4ebdb172dd..88c3fa57c1 100755 --- a/ci/scripts/run_ci_local.sh +++ b/ci/scripts/run_ci_local.sh @@ -49,7 +49,7 @@ GIT_BRANCH=$(git branch --show-current) GIT_COMMIT=$(git log -n 1 --pretty=format:%H) LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp} -CONTAINER_VER=${CONTAINER_VER:-230801} +CONTAINER_VER=${CONTAINER_VER:-230828} CUDA_VER=${CUDA_VER:-11.8} DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} diff --git a/docker/conda/environments/cuda11.8_examples.yml b/docker/conda/environments/cuda11.8_examples.yml index a878484df7..b55777be54 100644 --- a/docker/conda/environments/cuda11.8_examples.yml +++ b/docker/conda/environments/cuda11.8_examples.yml @@ -23,18 +23,14 @@ channels: - rapidsai - nvidia - conda-forge + - dglteam/label/cu118 dependencies: - boto3 - - chardet=5.0.0 - cuml=23.06 - dask>=2023.1.1 + - dgl=1.0.2 - dill=0.3.6 - distributed>=2023.1.1 - mlflow>=2.2.1,<3 - papermill=2.3.4 - s3fs>=2023.6 - - pip - - wrapt=1.14.1 # ver 1.15 breaks the keras model used by the gnn_fraud_detection_pipeline - - pip: - # tensorflow exists in conda-forge but is tied to CUDA-11.3 - - tensorflow==2.12.0 diff --git a/examples/gnn_fraud_detection_pipeline/README.md b/examples/gnn_fraud_detection_pipeline/README.md index 1a2b0ed3a3..f7d2d005dc 100644 --- a/examples/gnn_fraud_detection_pipeline/README.md +++ b/examples/gnn_fraud_detection_pipeline/README.md @@ -22,11 +22,8 @@ Prior to running the GNN fraud detection pipeline, additional requirements must ```bash mamba env update -n ${CONDA_DEFAULT_ENV} -f examples/gnn_fraud_detection_pipeline/requirements.yml -pip install --ignore-requires-python stellargraph==1.2.1 ``` -> **Note**: The `--ignore-requires-python` is needed because Stellargraph only officially supports Python versions prior to 3.9 ([stellargraph/stellargraph#1960](https://github.com/stellargraph/stellargraph/issues/1960)). - ## Running ##### Setup Env Variable @@ -44,23 +41,22 @@ python run.py --help Usage: run.py [OPTIONS] Options: - --num_threads INTEGER RANGE Number of internal pipeline threads to use + --num_threads INTEGER RANGE Number of internal pipeline threads to use. + [x>=1] --pipeline_batch_size INTEGER RANGE Internal batch size for the pipeline. Can be much larger than the model batch size. Also - used for Kafka consumers - + used for Kafka consumers. [x>=1] --model_max_batch_size INTEGER RANGE - Max batch size to use for the model - --input_file PATH Input filepath [required] + Max batch size to use for the model. [x>=1] + --model_fea_length INTEGER RANGE + Features length to use for the model. + [x>=1] + --input_file PATH Input data filepath. [required] + --training_file PATH Training data filepath. [required] + --model_dir PATH Trained model directory path [required] --output_file TEXT The path to the file where the inference output will be saved. - --training_file PATH Training data file [required] - --model_fea_length INTEGER RANGE - Features length to use for the model - --model-xgb-file PATH The name of the XGB model that is deployed - --model-hinsage-file PATH The name of the trained HinSAGE model file path - --help Show this message and exit. ``` @@ -71,35 +67,41 @@ cd ${MORPHEUS_ROOT}/examples/gnn_fraud_detection_pipeline python run.py ``` ``` +====Registering Pipeline==== ====Building Pipeline==== -Added source: +Graph construction rate: 0 messages [00:00, ? me====Building Pipeline Complete!==== +Inference rate: 0 messages [00:00, ? messages/s]====Registering Pipeline Complete!==== +====Starting Pipeline==== +====Pipeline Started==== 0 messages [00:00, ? messages/s] +====Building Segment: linear_segment_0====ges/s] +Added source: └─> morpheus.MessageMeta -Added stage: +Added stage: └─ morpheus.MessageMeta -> morpheus.MultiMessage Added stage: └─ morpheus.MultiMessage -> stages.FraudGraphMultiMessage -Added stage: +Added stage: └─ stages.FraudGraphMultiMessage -> stages.FraudGraphMultiMessage -Added stage: +Added stage: └─ stages.FraudGraphMultiMessage -> stages.GraphSAGEMultiMessage -Added stage: +Added stage: └─ stages.GraphSAGEMultiMessage -> stages.GraphSAGEMultiMessage -Added stage: +Added stage: └─ stages.GraphSAGEMultiMessage -> morpheus.MultiMessage -Added stage: +Added stage: └─ morpheus.MultiMessage -> morpheus.MultiMessage -Added stage: - └─ morpheus.MultiMessage -> pandas.DataFrame -Added stage: - └─ pandas.DataFrame -> pandas.DataFrame -Added stage: - └─ pandas.DataFrame -> pandas.DataFrame -====Building Pipeline Complete!==== -====Pipeline Started==== -Graph construction rate[Complete]: 265messages [00:00, 1590.22messages/s] -Inference rate[Complete]: 265messages [00:01, 150.23messages/s] -Add classification rate[Complete]: 265messages [00:01, 147.11messages/s] -Serialize rate[Complete]: 265messages [00:01, 142.31messages/s] +Added stage: + └─ morpheus.MultiMessage -> morpheus.MessageMeta +Added stage: + └─ morpheus.MessageMeta -> morpheus.MessageMeta +Added stage: + └─ morpheus.MessageMeta -> morpheus.MessageMeta +====Building Segment Complete!==== +Graph construction rate[Complete]: 265 messages [00:00, 1218.88 messages/s] +Inference rate[Complete]: 265 messages [00:01, 174.04 messages/s] +Add classification rate[Complete]: 265 messages [00:01, 170.69 messages/s] +Serialize rate[Complete]: 265 messages [00:01, 166.36 messages/s] +====Pipeline Complete==== ``` ### CLI Example @@ -118,9 +120,8 @@ morpheus --log_level INFO \ deserialize \ fraud-graph-construction --training_file examples/gnn_fraud_detection_pipeline/training.csv \ monitor --description "Graph construction rate" \ - gnn-fraud-sage --model_hinsage_file examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt \ + gnn-fraud-sage --model_dir examples/gnn_fraud_detection_pipeline/model/ \ monitor --description "Inference rate" \ - gnn-fraud-classification --model_xgb_file examples/gnn_fraud_detection_pipeline/model/xgb-model.pt \ monitor --description "Add classification rate" \ serialize \ to-file --filename "output.csv" --overwrite diff --git a/examples/gnn_fraud_detection_pipeline/model/graph.pkl b/examples/gnn_fraud_detection_pipeline/model/graph.pkl new file mode 100644 index 0000000000..6d5706fc4e Binary files /dev/null and b/examples/gnn_fraud_detection_pipeline/model/graph.pkl differ diff --git a/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/keras_metadata.pb b/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/keras_metadata.pb deleted file mode 100644 index 9f55d93262..0000000000 --- a/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/keras_metadata.pb +++ /dev/null @@ -1,30 +0,0 @@ - -root"_tf_keras_network*{"name": "model_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "class_name": "Functional", "config": {"name": "model_1", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_4"}, "name": "input_4", "inbound_nodes": []}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_5"}, "name": "input_5", "inbound_nodes": []}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}, "name": "input_2", "inbound_nodes": []}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_3"}, "name": "input_3", "inbound_nodes": []}, {"class_name": "Reshape", "config": {"name": "reshape_2", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "name": "reshape_2", "inbound_nodes": [[["input_4", 0, 0, {}]]]}, {"class_name": "Reshape", "config": {"name": "reshape_3", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "name": "reshape_3", "inbound_nodes": [[["input_5", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 1, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_1"}, "name": "input_1", "inbound_nodes": []}, {"class_name": "Reshape", "config": {"name": "reshape", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "name": "reshape", "inbound_nodes": [[["input_2", 0, 0, {}]]]}, {"class_name": "Reshape", "config": {"name": "reshape_1", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "name": "reshape_1", "inbound_nodes": [[["input_3", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_4", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_4", "inbound_nodes": [[["input_2", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_3", "inbound_nodes": [[["reshape_2", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_6", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_6", "inbound_nodes": [[["input_3", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_5", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_5", "inbound_nodes": [[["reshape_3", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_2", "inbound_nodes": [[["input_1", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout", "inbound_nodes": [[["reshape", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_1", "inbound_nodes": [[["reshape_1", 0, 0, {}]]]}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_1", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_1", "inbound_nodes": [[["dropout_4", 0, 0, {}], ["dropout_3", 0, 0, {}]]]}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_2", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_2", "inbound_nodes": [[["dropout_6", 0, 0, {}], ["dropout_5", 0, 0, {}]]]}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator", "inbound_nodes": [[["dropout_2", 0, 0, {}], ["dropout", 0, 0, {}], ["dropout_1", 0, 0, {}]]]}, {"class_name": "Reshape", "config": {"name": "reshape_4", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "name": "reshape_4", "inbound_nodes": [[["mean_hin_aggregator_1", 0, 0, {}]]]}, {"class_name": "Reshape", "config": {"name": "reshape_5", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "name": "reshape_5", "inbound_nodes": [[["mean_hin_aggregator_2", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_9", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_9", "inbound_nodes": [[["mean_hin_aggregator", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_7", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_7", "inbound_nodes": [[["reshape_4", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_8", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_8", "inbound_nodes": [[["reshape_5", 0, 0, {}]]]}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_3", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "linear", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_3", "inbound_nodes": [[["dropout_9", 0, 0, {}], ["dropout_7", 0, 0, {}], ["dropout_8", 0, 0, {}]]]}, {"class_name": "Reshape", "config": {"name": "reshape_6", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [64]}}, "name": "reshape_6", "inbound_nodes": [[["mean_hin_aggregator_3", 0, 0, {}]]]}, {"class_name": "Lambda", "config": {"name": "lambda", "trainable": true, "dtype": "float32", "function": {"class_name": "__tuple__", "items": ["4wEAAAAAAAAAAAAAAAEAAAAEAAAAUwAAAHMOAAAAdABqAXwAZAFkAo0CUwApA07p/////6kB2gRh\neGlzKQLaAUvaDGwyX25vcm1hbGl6ZakB2gF4qQByCAAAAPpQL29wdC9jb25kYS9lbnZzL3JhcGlk\ncy9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RlbGxhcmdyYXBoL2xheWVyL2hpbnNhZ2Uu\ncHnaCDxsYW1iZGE+ZgEAAPMAAAAA\n", null, null]}, "function_type": "lambda", "module": "stellargraph.layer.hinsage", "output_shape": null, "output_shape_type": "raw", "output_shape_module": null, "arguments": {}}, "name": "lambda", "inbound_nodes": [[["reshape_6", 0, 0, {}]]]}], "input_layers": [["input_1", 0, 0], ["input_2", 0, 0], ["input_3", 0, 0], ["input_4", 0, 0], ["input_5", 0, 0]], "output_layers": [["lambda", 0, 0]]}, "shared_object_id": 35, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 1, 111]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}, {"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}, {"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}, {"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}, {"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": [{"class_name": "TensorShape", "items": [null, 1, 111]}, {"class_name": "TensorShape", "items": [null, 2, 1]}, {"class_name": "TensorShape", "items": [null, 2, 1]}, {"class_name": "TensorShape", "items": [null, 64, 111]}, {"class_name": "TensorShape", "items": [null, 64, 111]}], "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 1, 111]}, "float32", "input_1"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 2, 1]}, "float32", "input_2"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 2, 1]}, "float32", "input_3"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 64, 111]}, "float32", "input_4"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 64, 111]}, "float32", "input_5"]}]], {}]}, "save_spec": [{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 1, 111]}, "float32", "input_1"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 2, 1]}, "float32", "input_2"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 2, 1]}, "float32", "input_3"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 64, 111]}, "float32", "input_4"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 64, 111]}, "float32", "input_5"]}], "keras_version": "2.9.0", "backend": "tensorflow", "model_config": {"class_name": "Functional", "config": {"name": "model_1", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_4"}, "name": "input_4", "inbound_nodes": [], "shared_object_id": 0}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_5"}, "name": "input_5", "inbound_nodes": [], "shared_object_id": 1}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}, "name": "input_2", "inbound_nodes": [], "shared_object_id": 2}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_3"}, "name": "input_3", "inbound_nodes": [], "shared_object_id": 3}, {"class_name": "Reshape", "config": {"name": "reshape_2", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "name": "reshape_2", "inbound_nodes": [[["input_4", 0, 0, {}]]], "shared_object_id": 4}, {"class_name": "Reshape", "config": {"name": "reshape_3", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "name": "reshape_3", "inbound_nodes": [[["input_5", 0, 0, {}]]], "shared_object_id": 5}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 1, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_1"}, "name": "input_1", "inbound_nodes": [], "shared_object_id": 6}, {"class_name": "Reshape", "config": {"name": "reshape", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "name": "reshape", "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 7}, {"class_name": "Reshape", "config": {"name": "reshape_1", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "name": "reshape_1", "inbound_nodes": [[["input_3", 0, 0, {}]]], "shared_object_id": 8}, {"class_name": "Dropout", "config": {"name": "dropout_4", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_4", "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 9}, {"class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_3", "inbound_nodes": [[["reshape_2", 0, 0, {}]]], "shared_object_id": 10}, {"class_name": "Dropout", "config": {"name": "dropout_6", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_6", "inbound_nodes": [[["input_3", 0, 0, {}]]], "shared_object_id": 11}, {"class_name": "Dropout", "config": {"name": "dropout_5", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_5", "inbound_nodes": [[["reshape_3", 0, 0, {}]]], "shared_object_id": 12}, {"class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_2", "inbound_nodes": [[["input_1", 0, 0, {}]]], "shared_object_id": 13}, {"class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout", "inbound_nodes": [[["reshape", 0, 0, {}]]], "shared_object_id": 14}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_1", "inbound_nodes": [[["reshape_1", 0, 0, {}]]], "shared_object_id": 15}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_1", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 16}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 17}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_1", "inbound_nodes": [[["dropout_4", 0, 0, {}], ["dropout_3", 0, 0, {}]]], "shared_object_id": 18}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_2", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 19}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 20}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_2", "inbound_nodes": [[["dropout_6", 0, 0, {}], ["dropout_5", 0, 0, {}]]], "shared_object_id": 21}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 22}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 23}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator", "inbound_nodes": [[["dropout_2", 0, 0, {}], ["dropout", 0, 0, {}], ["dropout_1", 0, 0, {}]]], "shared_object_id": 24}, {"class_name": "Reshape", "config": {"name": "reshape_4", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "name": "reshape_4", "inbound_nodes": [[["mean_hin_aggregator_1", 0, 0, {}]]], "shared_object_id": 25}, {"class_name": "Reshape", "config": {"name": "reshape_5", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "name": "reshape_5", "inbound_nodes": [[["mean_hin_aggregator_2", 0, 0, {}]]], "shared_object_id": 26}, {"class_name": "Dropout", "config": {"name": "dropout_9", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_9", "inbound_nodes": [[["mean_hin_aggregator", 0, 0, {}]]], "shared_object_id": 27}, {"class_name": "Dropout", "config": {"name": "dropout_7", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_7", "inbound_nodes": [[["reshape_4", 0, 0, {}]]], "shared_object_id": 28}, {"class_name": "Dropout", "config": {"name": "dropout_8", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_8", "inbound_nodes": [[["reshape_5", 0, 0, {}]]], "shared_object_id": 29}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_3", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "linear", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 30}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 31}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_3", "inbound_nodes": [[["dropout_9", 0, 0, {}], ["dropout_7", 0, 0, {}], ["dropout_8", 0, 0, {}]]], "shared_object_id": 32}, {"class_name": "Reshape", "config": {"name": "reshape_6", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [64]}}, "name": "reshape_6", "inbound_nodes": [[["mean_hin_aggregator_3", 0, 0, {}]]], "shared_object_id": 33}, {"class_name": "Lambda", "config": {"name": "lambda", "trainable": true, "dtype": "float32", "function": {"class_name": "__tuple__", "items": ["4wEAAAAAAAAAAAAAAAEAAAAEAAAAUwAAAHMOAAAAdABqAXwAZAFkAo0CUwApA07p/////6kB2gRh\neGlzKQLaAUvaDGwyX25vcm1hbGl6ZakB2gF4qQByCAAAAPpQL29wdC9jb25kYS9lbnZzL3JhcGlk\ncy9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RlbGxhcmdyYXBoL2xheWVyL2hpbnNhZ2Uu\ncHnaCDxsYW1iZGE+ZgEAAPMAAAAA\n", null, null]}, "function_type": "lambda", "module": "stellargraph.layer.hinsage", "output_shape": null, "output_shape_type": "raw", "output_shape_module": null, "arguments": {}}, "name": "lambda", "inbound_nodes": [[["reshape_6", 0, 0, {}]]], "shared_object_id": 34}], "input_layers": [["input_1", 0, 0], ["input_2", 0, 0], ["input_3", 0, 0], ["input_4", 0, 0], ["input_5", 0, 0]], "output_layers": [["lambda", 0, 0]]}}}2 - root.layer-0"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_4", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_4"}}2 - root.layer-1"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_5", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_5"}}2 - root.layer-2"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_2", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}}2 - root.layer-3"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_3", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_3"}}2 - root.layer-4"_tf_keras_layer*{"name": "reshape_2", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_2", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "inbound_nodes": [[["input_4", 0, 0, {}]]], "shared_object_id": 4}2 - root.layer-5"_tf_keras_layer*{"name": "reshape_3", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_3", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "inbound_nodes": [[["input_5", 0, 0, {}]]], "shared_object_id": 5}2 - root.layer-6"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_1", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 1, 111]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 1, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_1"}}2 - root.layer-7"_tf_keras_layer*{"name": "reshape", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 7}2 -  root.layer-8"_tf_keras_layer*{"name": "reshape_1", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_1", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "inbound_nodes": [[["input_3", 0, 0, {}]]], "shared_object_id": 8}2 - - root.layer-9"_tf_keras_layer*{"name": "dropout_4", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_4", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 9, "build_input_shape": {"class_name": "TensorShape", "items": [null, 2, 1]}}2 -  root.layer-10"_tf_keras_layer*{"name": "dropout_3", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape_2", 0, 0, {}]]], "shared_object_id": 10, "build_input_shape": {"class_name": "TensorShape", "items": [null, 2, 32, 111]}}2 -  root.layer-11"_tf_keras_layer*{"name": "dropout_6", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_6", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["input_3", 0, 0, {}]]], "shared_object_id": 11, "build_input_shape": {"class_name": "TensorShape", "items": [null, 2, 1]}}2 -  root.layer-12"_tf_keras_layer*{"name": "dropout_5", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_5", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape_3", 0, 0, {}]]], "shared_object_id": 12, "build_input_shape": {"class_name": "TensorShape", "items": [null, 2, 32, 111]}}2 - root.layer-13"_tf_keras_layer*{"name": "dropout_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["input_1", 0, 0, {}]]], "shared_object_id": 13, "build_input_shape": {"class_name": "TensorShape", "items": [null, 1, 111]}}2 - root.layer-14"_tf_keras_layer*{"name": "dropout", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape", 0, 0, {}]]], "shared_object_id": 14, "build_input_shape": {"class_name": "TensorShape", "items": [null, 1, 2, 1]}}2 - root.layer-15"_tf_keras_layer*{"name": "dropout_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape_1", 0, 0, {}]]], "shared_object_id": 15, "build_input_shape": {"class_name": "TensorShape", "items": [null, 1, 2, 1]}}2 -root.layer_with_weights-0"_tf_keras_layer*{"name": "mean_hin_aggregator_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_1", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 16}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 17}, "bias_regularizer": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_4", 0, 0, {}], ["dropout_3", 0, 0, {}]]], "shared_object_id": 18, "build_input_shape": [{"class_name": "TensorShape", "items": [null, 2, 1]}, {"class_name": "TensorShape", "items": [null, 2, 32, 111]}]}2 -root.layer_with_weights-1"_tf_keras_layer*{"name": "mean_hin_aggregator_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_2", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 19}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 20}, "bias_regularizer": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_6", 0, 0, {}], ["dropout_5", 0, 0, {}]]], "shared_object_id": 21, "build_input_shape": [{"class_name": "TensorShape", "items": [null, 2, 1]}, {"class_name": "TensorShape", "items": [null, 2, 32, 111]}]}2 -root.layer_with_weights-2"_tf_keras_layer*{"name": "mean_hin_aggregator", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 22}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 23}, "bias_regularizer": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_2", 0, 0, {}], ["dropout", 0, 0, {}], ["dropout_1", 0, 0, {}]]], "shared_object_id": 24, "build_input_shape": [{"class_name": "TensorShape", "items": [null, 1, 111]}, {"class_name": "TensorShape", "items": [null, 1, 2, 1]}, {"class_name": "TensorShape", "items": [null, 1, 2, 1]}]}2 - root.layer-19"_tf_keras_layer*{"name": "reshape_4", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_4", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "inbound_nodes": [[["mean_hin_aggregator_1", 0, 0, {}]]], "shared_object_id": 25}2 - root.layer-20"_tf_keras_layer*{"name": "reshape_5", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_5", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "inbound_nodes": [[["mean_hin_aggregator_2", 0, 0, {}]]], "shared_object_id": 26}2 - root.layer-21"_tf_keras_layer*{"name": "dropout_9", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_9", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["mean_hin_aggregator", 0, 0, {}]]], "shared_object_id": 27, "build_input_shape": {"class_name": "TensorShape", "items": [null, 1, 64]}}2 - root.layer-22"_tf_keras_layer*{"name": "dropout_7", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_7", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape_4", 0, 0, {}]]], "shared_object_id": 28, "build_input_shape": {"class_name": "TensorShape", "items": [null, 1, 2, 64]}}2 - root.layer-23"_tf_keras_layer*{"name": "dropout_8", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_8", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape_5", 0, 0, {}]]], "shared_object_id": 29, "build_input_shape": {"class_name": "TensorShape", "items": [null, 1, 2, 64]}}2 -root.layer_with_weights-3"_tf_keras_layer*{"name": "mean_hin_aggregator_3", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_3", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "linear", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 30}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 31}, "bias_regularizer": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_9", 0, 0, {}], ["dropout_7", 0, 0, {}], ["dropout_8", 0, 0, {}]]], "shared_object_id": 32, "build_input_shape": [{"class_name": "TensorShape", "items": [null, 1, 64]}, {"class_name": "TensorShape", "items": [null, 1, 2, 64]}, {"class_name": "TensorShape", "items": [null, 1, 2, 64]}]}2 - root.layer-25"_tf_keras_layer*{"name": "reshape_6", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_6", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [64]}}, "inbound_nodes": [[["mean_hin_aggregator_3", 0, 0, {}]]], "shared_object_id": 33}2 - root.layer-26"_tf_keras_layer*{"name": "lambda", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Lambda", "config": {"name": "lambda", "trainable": true, "dtype": "float32", "function": {"class_name": "__tuple__", "items": ["4wEAAAAAAAAAAAAAAAEAAAAEAAAAUwAAAHMOAAAAdABqAXwAZAFkAo0CUwApA07p/////6kB2gRh\neGlzKQLaAUvaDGwyX25vcm1hbGl6ZakB2gF4qQByCAAAAPpQL29wdC9jb25kYS9lbnZzL3JhcGlk\ncy9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RlbGxhcmdyYXBoL2xheWVyL2hpbnNhZ2Uu\ncHnaCDxsYW1iZGE+ZgEAAPMAAAAA\n", null, null]}, "function_type": "lambda", "module": "stellargraph.layer.hinsage", "output_shape": null, "output_shape_type": "raw", "output_shape_module": null, "arguments": {}}, "inbound_nodes": [[["reshape_6", 0, 0, {}]]], "shared_object_id": 34}2 \ No newline at end of file diff --git a/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/saved_model.pb b/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/saved_model.pb deleted file mode 100644 index 379570b809..0000000000 Binary files a/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/saved_model.pb and /dev/null differ diff --git a/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/variables/variables.data-00000-of-00001 b/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/variables/variables.data-00000-of-00001 deleted file mode 100644 index 636af39730..0000000000 Binary files a/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/variables/variables.data-00000-of-00001 and /dev/null differ diff --git a/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/variables/variables.index b/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/variables/variables.index deleted file mode 100644 index 18c0896df9..0000000000 Binary files a/examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt/variables/variables.index and /dev/null differ diff --git a/examples/gnn_fraud_detection_pipeline/model/hyperparams.pkl b/examples/gnn_fraud_detection_pipeline/model/hyperparams.pkl new file mode 100644 index 0000000000..7c87766fb0 Binary files /dev/null and b/examples/gnn_fraud_detection_pipeline/model/hyperparams.pkl differ diff --git a/examples/gnn_fraud_detection_pipeline/model/model.pt b/examples/gnn_fraud_detection_pipeline/model/model.pt new file mode 100644 index 0000000000..29ca394789 Binary files /dev/null and b/examples/gnn_fraud_detection_pipeline/model/model.pt differ diff --git a/examples/gnn_fraud_detection_pipeline/model/xgb-model.pt b/examples/gnn_fraud_detection_pipeline/model/xgb-model.pt deleted file mode 100644 index cb2abddbea..0000000000 Binary files a/examples/gnn_fraud_detection_pipeline/model/xgb-model.pt and /dev/null differ diff --git a/examples/gnn_fraud_detection_pipeline/model/xgb.pt b/examples/gnn_fraud_detection_pipeline/model/xgb.pt new file mode 100644 index 0000000000..24e5212729 Binary files /dev/null and b/examples/gnn_fraud_detection_pipeline/model/xgb.pt differ diff --git a/examples/gnn_fraud_detection_pipeline/requirements.yml b/examples/gnn_fraud_detection_pipeline/requirements.yml index 66fa035b27..01f641c047 100644 --- a/examples/gnn_fraud_detection_pipeline/requirements.yml +++ b/examples/gnn_fraud_detection_pipeline/requirements.yml @@ -17,12 +17,7 @@ channels: - rapidsai - nvidia - conda-forge + - dglteam/label/cu118 dependencies: - - chardet=5.0.0 - cuml=23.06 - - dask>=2023.1.1 - - distributed>=2023.1.1 - - pip - - pip: - # tensorflow exists in conda-forge but is tied to CUDA-11.3 - - tensorflow==2.12.0 + - dgl=1.0.2 diff --git a/examples/gnn_fraud_detection_pipeline/run.py b/examples/gnn_fraud_detection_pipeline/run.py index 3c3078ef7a..6fe0cfb804 100644 --- a/examples/gnn_fraud_detection_pipeline/run.py +++ b/examples/gnn_fraud_detection_pipeline/run.py @@ -27,6 +27,7 @@ from morpheus.stages.postprocess.serialize_stage import SerializeStage from morpheus.stages.preprocess.deserialize_stage import DeserializeStage from morpheus.utils.logger import configure_logging +# pylint: disable=no-name-in-module from stages.classification_stage import ClassificationStage from stages.graph_construction_stage import FraudGraphConstructionStage from stages.graph_sage_stage import GraphSAGEStage @@ -60,48 +61,39 @@ ) @click.option( "--input_file", - type=click.Path(exists=True, readable=True), + type=click.Path(exists=True, readable=True, dir_okay=False), default="validation.csv", required=True, help="Input data filepath.", ) @click.option( "--training_file", - type=click.Path(exists=True, readable=True), + type=click.Path(exists=True, readable=True, dir_okay=False), default="training.csv", required=True, help="Training data filepath.", ) @click.option( - "--model-hinsage-file", - type=click.Path(exists=True, readable=True), - default="model/hinsage-model.pt", + "--model_dir", + type=click.Path(exists=True, readable=True, file_okay=False, dir_okay=True), + default="model", required=True, - help="Trained hinsage model filepath.", -) -@click.option( - "--model-xgb-file", - type=click.Path(exists=True, readable=True), - default="model/xgb-model.pt", - required=True, - help="Trained xgb model filepath.", + help="Path to trained Hinsage & XGB models.", ) @click.option( "--output_file", + type=click.Path(dir_okay=False), default="output.csv", help="The path to the file where the inference output will be saved.", ) -def run_pipeline( - num_threads, - pipeline_batch_size, - model_max_batch_size, - model_fea_length, - input_file, - training_file, - model_hinsage_file, - model_xgb_file, - output_file, -): +def run_pipeline(num_threads, + pipeline_batch_size, + model_max_batch_size, + model_fea_length, + input_file, + training_file, + model_dir, + output_file): # Enable the default logger. configure_logging(log_level=logging.INFO) @@ -140,12 +132,12 @@ def run_pipeline( pipeline.add_stage(MonitorStage(config, description="Graph construction rate")) # Add a sage inference stage. - pipeline.add_stage(GraphSAGEStage(config, model_hinsage_file)) + pipeline.add_stage(GraphSAGEStage(config, model_dir)) pipeline.add_stage(MonitorStage(config, description="Inference rate")) # Add classification stage. # This stage adds detected classifications to each message. - pipeline.add_stage(ClassificationStage(config, model_xgb_file)) + pipeline.add_stage(ClassificationStage(config, os.path.join(model_dir, "xgb.pt"))) pipeline.add_stage(MonitorStage(config, description="Add classification rate")) # Add a serialize stage. diff --git a/examples/gnn_fraud_detection_pipeline/stages/classification_stage.py b/examples/gnn_fraud_detection_pipeline/stages/classification_stage.py index 3abdcd5561..029011d9b8 100644 --- a/examples/gnn_fraud_detection_pipeline/stages/classification_stage.py +++ b/examples/gnn_fraud_detection_pipeline/stages/classification_stage.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import typing - import mrc from mrc.core import operators as ops @@ -55,13 +53,13 @@ def __init__(self, c: Config, model_xgb_file: str): def name(self) -> str: return "gnn-fraud-classification" - def accepted_types(self) -> typing.Tuple: + def accepted_types(self) -> (GraphSAGEMultiMessage, ): return (GraphSAGEMultiMessage, ) - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: return False - def _process_message(self, message: GraphSAGEMultiMessage): + def _process_message(self, message: GraphSAGEMultiMessage) -> GraphSAGEMultiMessage: ind_emb_columns = message.get_meta(message.inductive_embedding_column_names) message.set_meta("node_id", message.node_identifiers) diff --git a/examples/gnn_fraud_detection_pipeline/stages/graph_construction_stage.py b/examples/gnn_fraud_detection_pipeline/stages/graph_construction_stage.py index cae0a6a81b..37cc47ca84 100644 --- a/examples/gnn_fraud_detection_pipeline/stages/graph_construction_stage.py +++ b/examples/gnn_fraud_detection_pipeline/stages/graph_construction_stage.py @@ -15,11 +15,10 @@ import dataclasses import pathlib -import typing +import dgl import mrc -import networkx as nx -import pandas as pd +import torch from mrc.core import operators as ops import cudf @@ -32,26 +31,32 @@ from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.pipeline.stream_pair import StreamPair +from .model import build_fsi_graph +from .model import prepare_data + @dataclasses.dataclass class FraudGraphMultiMessage(MultiMessage): - graph: "stellargraph.StellarGraph" def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, - graph: "stellargraph.StellarGraph"): + graph: dgl.DGLHeteroGraph, + node_features=torch.tensor, + test_index: torch.tensor): super().__init__(meta=meta, mess_offset=mess_offset, mess_count=mess_count) self.graph = graph + self.node_features = node_features + self.test_index = test_index @register_stage("fraud-graph-construction", modes=[PipelineModes.OTHER]) class FraudGraphConstructionStage(SinglePortStage): - def __init__(self, c: Config, training_file: pathlib.Path): + def __init__(self, config: Config, training_file: pathlib.Path): """ Create a fraud-graph-construction stage @@ -62,7 +67,7 @@ def __init__(self, c: Config, training_file: pathlib.Path): training_file : pathlib.Path, exists = True, dir_okay = False A CSV training file to load to seed the graph """ - super().__init__(c) + super().__init__(config) self._training_data = cudf.read_csv(training_file) self._column_names = self._training_data.columns.values.tolist() @@ -70,60 +75,28 @@ def __init__(self, c: Config, training_file: pathlib.Path): def name(self) -> str: return "fraud-graph-construction" - def accepted_types(self) -> typing.Tuple: + def accepted_types(self) -> (MultiMessage, ): return (MultiMessage, ) - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: return False - @staticmethod - def _graph_construction(nodes, edges, node_features) -> "stellargraph.StellarGraph": - - from stellargraph import StellarGraph - - g_nx = nx.Graph() - - # add nodes - for key, values in nodes.items(): - g_nx.add_nodes_from(values, ntype=key) - # add edges - for edge in edges: - g_nx.add_edges_from(edge) - - return StellarGraph.from_networkx(g_nx, node_type_attr='ntype', node_features=node_features) - - @staticmethod - def _build_graph_features(dataset: pd.DataFrame) -> "stellargraph.StellarGraph": - - nodes = { - "client": dataset.client_node, - "merchant": dataset.merchant_node, - "transaction": dataset.index, - } - - edges = [ - zip(dataset.client_node, dataset.index), - zip(dataset.merchant_node, dataset.index), - ] + def _process_message(self, message: MultiMessage) -> FraudGraphMultiMessage: - transaction_node_data = dataset.drop(["client_node", "merchant_node", "fraud_label", "index"], axis=1) - client_node_data = pd.DataFrame([1] * len(dataset.client_node.unique())).set_index(dataset.client_node.unique()) - merchant_node_data = pd.DataFrame([1] * len(dataset.merchant_node.unique())).set_index( - dataset.merchant_node.unique()) + _, _, _, test_index, _, graph_data = prepare_data(self._training_data, message.get_meta(self._column_names)) - node_features = { - "transaction": transaction_node_data, - "client": client_node_data, - "merchant": merchant_node_data, - } + # meta columns to remove as node features + meta_cols = ['client_node', 'merchant_node', 'index'] + graph, node_features = build_fsi_graph(graph_data, meta_cols) - return FraudGraphConstructionStage._graph_construction(nodes, edges, node_features) + # Convert to torch.tensor from cupy + test_index = torch.from_dlpack(test_index.values.toDlpack()).long() + node_features = node_features.float() - def _process_message(self, message: MultiMessage): - graph_data = cudf.concat([self._training_data, message.get_meta(self._column_names)]) - graph_data = graph_data.set_index(graph_data['index']) - graph = FraudGraphConstructionStage._build_graph_features(graph_data.to_pandas()) - return FraudGraphMultiMessage.from_message(message, graph=graph) + return FraudGraphMultiMessage.from_message(message, + graph=graph, + node_features=node_features.float(), + test_index=test_index) def _build_single(self, builder: mrc.Builder, input_stream: StreamPair) -> StreamPair: node = builder.make_node(self.unique_name, ops.map(self._process_message)) diff --git a/examples/gnn_fraud_detection_pipeline/stages/graph_sage_stage.py b/examples/gnn_fraud_detection_pipeline/stages/graph_sage_stage.py index 90c3ebd5df..bfa2eb62ec 100644 --- a/examples/gnn_fraud_detection_pipeline/stages/graph_sage_stage.py +++ b/examples/gnn_fraud_detection_pipeline/stages/graph_sage_stage.py @@ -14,10 +14,8 @@ # limitations under the License. import dataclasses -import typing import mrc -import numpy as np from mrc.core import operators as ops import cudf @@ -31,20 +29,21 @@ from morpheus.pipeline.stream_pair import StreamPair from .graph_construction_stage import FraudGraphMultiMessage +from .model import load_model @dataclasses.dataclass class GraphSAGEMultiMessage(MultiMessage): - node_identifiers: typing.List[int] - inductive_embedding_column_names: typing.List[str] + node_identifiers: list[int] + inductive_embedding_column_names: list[str] def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, - node_identifiers: typing.List[int], - inductive_embedding_column_names: typing.List[str]): + node_identifiers: list[int], + inductive_embedding_column_names: list[str]): super().__init__(meta=meta, mess_offset=mess_offset, mess_count=mess_count) self.node_identifiers = node_identifiers @@ -55,24 +54,15 @@ def __init__(self, class GraphSAGEStage(SinglePortStage): def __init__(self, - c: Config, - model_hinsage_file: str, - batch_size: int = 5, - sample_size: typing.List[int] = None, + config: Config, + model_dir: str, + batch_size: int = 100, record_id: str = "index", target_node: str = "transaction"): - super().__init__(c) + super().__init__(config) - if (sample_size is None): - sample_size = [2, 32] - - # Must import stellargraph before loading the model - import stellargraph.mapper # noqa: F401 # pylint:disable=unused-import - import tensorflow as tf - - self._keras_model = tf.keras.models.load_model(model_hinsage_file) + self._dgl_model, _, __ = load_model(model_dir) self._batch_size = batch_size - self._sample_size = list(sample_size) self._record_id = record_id self._target_node = target_node @@ -80,35 +70,23 @@ def __init__(self, def name(self) -> str: return "gnn-fraud-sage" - def accepted_types(self) -> typing.Tuple: + def accepted_types(self) -> (FraudGraphMultiMessage, ): return (FraudGraphMultiMessage, ) - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: return False - def _inductive_step_hinsage( - self, - graph, - trained_model, - node_identifiers, - ): - - from stellargraph.mapper import HinSAGENodeGenerator - - # perform inductive learning from trained graph model - # The mapper feeds data from sampled subgraph to HinSAGE model - generator = HinSAGENodeGenerator(graph, self._batch_size, self._sample_size, head_node_type=self._target_node) - test_gen_not_shuffled = generator.flow(node_identifiers, shuffle=False) + def _process_message(self, message: FraudGraphMultiMessage) -> GraphSAGEMultiMessage: - inductive_emb = np.concatenate([trained_model.predict(row[0]) for row in test_gen_not_shuffled]) - inductive_emb = cudf.DataFrame(inductive_emb, index=node_identifiers) - - return inductive_emb - - def _process_message(self, message: FraudGraphMultiMessage): node_identifiers = list(message.get_meta(self._record_id).to_pandas()) - inductive_embedding = self._inductive_step_hinsage(message.graph, self._keras_model, node_identifiers) + # Perform inference + inductive_embedding, _ = self._dgl_model.inference(message.graph, + message.node_features, + message.test_index, + batch_size=self._batch_size) + + inductive_embedding = cudf.DataFrame(inductive_embedding) # Rename the columns to be more descriptive inductive_embedding.rename(lambda x: "ind_emb_" + str(x), axis=1, inplace=True) diff --git a/examples/gnn_fraud_detection_pipeline/stages/model.py b/examples/gnn_fraud_detection_pipeline/stages/model.py new file mode 100644 index 0000000000..36ce64b6aa --- /dev/null +++ b/examples/gnn_fraud_detection_pipeline/stages/model.py @@ -0,0 +1,510 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pickle + +import cupy +import dgl +import torch +from dgl import nn as dglnn +from torch import nn +from torch.nn import functional as F + +import cudf as cf + + +class BaseHeteroGraph(nn.Module): + """ + Base class for Heterogeneous Graph Neural Network (GNN) models. + + Parameters + ---------- + input_graph : dgl.DGLHeteroGraph + The input graph on which the HeteroRGCN operates. It should be a heterogeneous graph. + embedding_size : int + The size of the node embeddings learned during training. + target : str, optional + The target attribute for which the node representations are learned. + """ + + def __init__(self, input_graph: dgl.DGLHeteroGraph, embedding_size: int, target: str): + + super().__init__() + self._target = target + + # categorical embeding + self.hetro_embedding = dglnn.HeteroEmbedding( + {ntype: input_graph.number_of_nodes(ntype) + for ntype in input_graph.ntypes if ntype != self._target}, + embedding_size) + + self.layers = nn.ModuleList() + + def forward(self, input_graph: dgl.DGLHeteroGraph, features: torch.tensor) -> (torch.tensor, torch.tensor): + + # Get embeddings for all none target node types. + h_dict = self.hetro_embedding( + {ntype: input_graph[0].nodes(ntype) + for ntype in self.hetro_embedding.embeds.keys()}) + + h_dict[self._target] = features + + # Forward pass to layers. + for i, layer in enumerate(self.layers[:-1]): + if i != 0: + h_dict = {k: F.leaky_relu(h) for k, h in h_dict.items()} + h_dict = layer(input_graph[i], h_dict) + + embedding = h_dict[self._target] + + return self.layers[-1](embedding), embedding + + def infer(self, input_graph: dgl.DGLHeteroGraph, features: torch.tensor) -> (torch.tensor, torch.tensor): + """Perform inference through forward pass + + Parameters + ---------- + input_graph : dgl.DGLHeteroGraph + input inference graph + features : torch.tensor + node features + + Returns + ------- + torch.tensor, torch.tensor + prediction, feature embedding + """ + + predictions, embedding = self(input_graph, features) + return nn.Sigmoid()(predictions), embedding + + @torch.no_grad() + def evaluate(self, eval_loader: dgl.dataloading.DataLoader, feature_tensors: torch.Tensor, + target_node: str) -> (torch.Tensor, torch.Tensor, torch.Tensor): + """Evaluate the specified model on the given evaluation input graph + + Parameters + ---------- + eval_loader : dgl.dataloading.DataLoader + DataLoader containing the evaluation dataset. + feature_tensors : torch.Tensor + The feature tensors corresponding to the evaluation data. + Shape: (num_samples, num_features). + target_node : str + The target node for evaluation, indicating the node of interest. + + Returns + ------- + (torch.Tensor, torch.Tensor, torch.Tensor) + A tuple containing numpy arrays of logits, eval seed and embeddings + """ + + self.eval() + eval_logits = [] + eval_seeds = [] + embedding = [] + + for _, output_nodes, blocks in eval_loader: + + seed = output_nodes[target_node] + + nid = blocks[0].srcnodes[target_node].data[dgl.NID] + input_features = feature_tensors[nid] + logits, embedd = self.infer(blocks, input_features) + eval_logits.append(logits.cpu().detach()) + eval_seeds.append(seed) + embedding.append(embedd) + + eval_logits = torch.cat(eval_logits) + eval_seeds = torch.cat(eval_seeds) + embedding = torch.cat(embedding) + return eval_logits, eval_seeds, embedding + + def inference(self, + input_graph: dgl.DGLHeteroGraph, + feature_tensors: torch.Tensor, + test_idx: torch.Tensor, + target_node: str = "transaction", + batch_size: int = 100) -> (torch.Tensor, torch.Tensor): + """ + Perform inference on a given model using the provided input graph and feature tensors. + + Parameters + ---------- + input_graph : dgl.DGLHeteroGraph + The input heterogeneous graph in DGL format. It represents the graph structure. + feature_tensors : torch.Tensor + The input feature tensors for nodes in the input graph. Each row corresponds to the features of a single + node. + test_idx : torch.Tensor + The indices of the nodes in the input graph that are used for testing and evaluation. + target_node : str, optional (default: "transaction") + The type of node for which inference will be performed. By default, it is set to "transaction". + batch_size : int, optional (default: 100) + The batch size used during inference to process data in mini-batches. + + Returns + ------- + test_embedding : torch.Tensor + The embedding of for the target nodes obtained from the model's inference. + test_seed: torch.Tensor + The seed of the target nodes used for inference. + """ + + # create sampler and test dataloaders + full_sampler = dgl.dataloading.MultiLayerNeighborSampler(fanouts=[4, 3]) + test_dataloader = dgl.dataloading.DataLoader(input_graph, {target_node: test_idx}, + full_sampler, + batch_size=batch_size, + shuffle=False, + drop_last=False, + num_workers=0) + _, test_seed, test_embedding = self.evaluate(test_dataloader, feature_tensors, target_node) + + return test_embedding, test_seed + + +class HeteroRGCN(BaseHeteroGraph): + """ + Heterogeneous Relational Graph Convolutional Network (HeteroRGCN) model. + + This class represents a Heterogeneous Relational Graph Convolutional Network (HeteroRGCN) + used for node representation learning in heterogeneous graphs. + + Parameters + ---------- + input_graph : dgl.DGLHeteroGraph + The input graph on which the HeteroRGCN operates. It should be a heterogeneous graph. + in_size : int + The input feature size for each node in the graph. + hidden_size : int + The number of hidden units or feature size of the nodes after the convolutional layers. + out_size : int + The output feature size for each node. This will be the final representation size of each node. + n_layers : int + The number of graph convolutional layers in the HeteroRGCN model. + embedding_size : int + The size of the node embeddings learned during training. + target : str, optional + The target attribute for which the node representations are learned. + Default is 'transaction'. + + Attributes + ---------- + input_graph : dgl.DGLHeteroGraph + The input graph on which the HeteroRGCN operates. + in_size : int + The input feature size for each node in the graph. + hidden_size : int + The number of hidden units or feature size of the nodes after the convolutional layers. + out_size : int + The output feature size for each node. This will be the final representation size of each node. + n_layers : int + The number of graph convolutional layers in the HeteroRGCN model. + embedding_size : int + The size of the node embeddings learned during training + target : str + The target attribute for which the node representations are learned. + + Methods + ------- + forward(input_graph, features) + Forward pass of the HeteroRGCN model. + + Notes + ----- + HeteroRGCN is a deep learning model designed for heterogeneous graphs. + It applies graph convolutional layers to learn representations of nodes in the graph. + + Examples + -------- + >>> input_graph = dgl.heterograph(...) + >>> in_size = 16 + >>> hidden_size = 32 + >>> out_size = 64 + >>> n_layers = 2 + >>> embedding_size = 128 + >>> target = 'transaction' + >>> model = HeteroRGCN(input_graph, in_size, hidden_size, out_size, n_layers, embedding_size, target='transaction') + """ + + def __init__(self, + input_graph: dgl.DGLHeteroGraph, + in_size: int, + hidden_size: int, + out_size: int, + n_layers: int, + embedding_size: int, + target: str = 'transaction'): + + super().__init__(input_graph=input_graph, embedding_size=embedding_size, target=target) + + # input size + in_sizes = { + rel: in_size if src_type == self._target else embedding_size + for src_type, + rel, + _ in input_graph.canonical_etypes + } + + self.layers.append( + dglnn.HeteroGraphConv({rel: dglnn.GraphConv(in_sizes[rel], hidden_size) + for rel in input_graph.etypes}, + aggregate='sum')) + + for _ in range(n_layers - 1): + self.layers.append( + dglnn.HeteroGraphConv({rel: dglnn.GraphConv(hidden_size, hidden_size) + for rel in input_graph.etypes}, + aggregate='sum')) + + # output layer + self.layers.append(nn.Linear(hidden_size, out_size)) + + +class HinSAGE(BaseHeteroGraph): + """ + Heterogeneous GraphSAGE (HinSAGE) module for graph-based semi-supervised learning. + + This module implements a variant of GraphSAGE (Graph Sample and Aggregated) for heterogeneous graphs, + where different types of nodes have distinct feature spaces. + + Parameters + ---------- + input_graph : dgl.DGLHeteroGraph + The input heterogeneous graph on which the HinSAGE model will operate. + in_size : int + The input feature size for each node type. This represents the dimensionality of the node features. + hidden_size : int + The size of the hidden layer(s) in the GraphSAGE model. + out_size : int + The output size for each node type. This represents the dimensionality of the output node embeddings. + n_layers : int + The number of GraphSAGE layers in the HinSAGE model. + embedding_size : int + The size of the final node embeddings after aggregation. This will be used as input for the downstream task. + target : str, optional + The target node type for the downstream task. Default is 'transaction'. + aggregator_type : str, optional + The type of aggregator to use for aggregation. Default is 'mean'. + + Methods + ------- + forward(input_graph, features) + Forward pass of the HinSAGE model. + + Notes + ----- + HinSAGE is designed for semi-supervised learning on heterogeneous graphs. The model generates node embeddings by + sampling and aggregating neighbor information from different node types in the graph. + + The target parameter specifies the node type for the downstream task. The model will only return embeddings for + nodes of the specified type. + + Examples + -------- + >>> input_graph = dgl.heterograph(...) # Replace ... with actual heterogeneous graph data + >>> in_size = 64 + >>> hidden_size = 128 + >>> out_size = 64 + >>> n_layers = 2 + >>> embedding_size = 256 + >>> target_node_type = 'transaction' + >>> hinsage_model = HinSAGE(input_graph, in_size, hidden_size, out_size, n_layers, embedding_size, target_node_type) + """ + + def __init__(self, + input_graph: dgl.DGLHeteroGraph, + in_size: int, + hidden_size: int, + out_size: int, + n_layers: int, + embedding_size: int, + target: str = 'transaction', + aggregator_type: str = 'mean'): + + super().__init__(input_graph=input_graph, embedding_size=embedding_size, target=target) + + # create input features + in_feats = {rel: embedding_size if rel != self._target else in_size for rel in input_graph.ntypes} + + self.layers.append( + dglnn.HeteroGraphConv( + { + rel: + dglnn.SAGEConv( + (in_feats[src_type], in_feats[v_type]), hidden_size, aggregator_type=aggregator_type) + for src_type, + rel, + v_type in input_graph.canonical_etypes + }, + aggregate='sum')) + + for _ in range(n_layers - 1): + self.layers.append( + dglnn.HeteroGraphConv( + { + rel: dglnn.SAGEConv( + hidden_size, + hidden_size, + aggregator_type=aggregator_type, + ) + for rel in input_graph.etypes + }, + aggregate='sum')) + + # output layer + self.layers.append(nn.Linear(hidden_size, out_size)) + + +def load_model(model_dir: str, + gnn_model: BaseHeteroGraph = HinSAGE, + device: torch.device = None) -> (BaseHeteroGraph, dgl.DGLHeteroGraph, dict): + """Load trained models from model directory + + Parameters + ---------- + model_dir : str + models directory path + gnn_model: BaseHeteroGraph + GNN model type either HeteroRGCN or HinSAGE. Default HinSAGE + device : torch.device, optional + The device where the model and tensors should be loaded, + by default torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + Returns + ------- + (nn.Module, dgl.DGLHeteroGraph, dict) + model, training graph, hyperparameter + """ + + with open(os.path.join(model_dir, "graph.pkl"), 'rb') as f: + graph = pickle.load(f) + with open(os.path.join(model_dir, 'hyperparams.pkl'), 'rb') as f: + hyperparameters = pickle.load(f) + + if device is None: + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + model = gnn_model(graph, + in_size=hyperparameters['in_size'], + hidden_size=hyperparameters['hidden_size'], + out_size=hyperparameters['out_size'], + n_layers=hyperparameters['n_layers'], + embedding_size=hyperparameters['embedding_size'], + target=hyperparameters['target_node']).to(device) + model.load_state_dict(torch.load(os.path.join(model_dir, 'model.pt'))) + + return model, graph, hyperparameters + + +def build_fsi_graph(train_data: cf.DataFrame, col_drop: list[str]) -> (dgl.DGLHeteroGraph, torch.Tensor): + """Build a heterogeneous graph from an edgelist and node index. + Parameters + ---------- + train_data : cudf.DataFrame + Training data containing node features. + col_drop : list + List of features to drop from the node features. + Returns + ------- + tuple + A tuple containing the following elements: + dgl.DGLHeteroGraph + The built DGL graph representing the heterogeneous graph. + torch.tensor + Normalized feature tensor after dropping specified columns. + Notes + ----- + This function takes the training data, represented as a cudf DataFrame, + and constructs a heterogeneous graph (DGLGraph) from the given edgelist + and node index. + + The `col_drop` list specifies which features should be dropped from the + node features to build the normalized feature tensor. + + Example + ------- + >>> import cudf + >>> train_data = cudf.DataFrame({'node_id': [1, 2, 3], + ... 'feature1': [0.1, 0.2, 0.3], + ... 'feature2': [0.4, 0.5, 0.6]}) + >>> col_drop = ['feature2'] + >>> graph, features = build_heterograph(train_data, col_drop) + """ + + feature_tensors = train_data.drop(col_drop, axis=1).values + feature_tensors = torch.from_dlpack(feature_tensors.toDlpack()) + feature_tensors = (feature_tensors - feature_tensors.mean(0, keepdim=True)) / (0.0001 + + feature_tensors.std(0, keepdim=True)) + # Create client, merchant, transaction node id tensors & move to torch.tensor + # col_drop column expected to be in ['client','merchant', 'transaction'] order to match + # torch.tensor_split order + client_tensor, merchant_tensor, transaction_tensor = torch.tensor_split( + torch.from_dlpack(train_data[col_drop].values.toDlpack()).long(), 3, dim=1) + + client_tensor, merchant_tensor, transaction_tensor = (client_tensor.view(-1), + merchant_tensor.view(-1), + transaction_tensor.view(-1)) + + edge_list = { + ('client', 'buy', 'transaction'): (client_tensor, transaction_tensor), + ('transaction', 'bought', 'client'): (transaction_tensor, client_tensor), + ('transaction', 'issued', 'merchant'): (transaction_tensor, merchant_tensor), + ('merchant', 'sell', 'transaction'): (merchant_tensor, transaction_tensor) + } + graph = dgl.heterograph(edge_list) + + return graph, feature_tensors + + +def prepare_data( + training_data: cf.DataFrame, + test_data: cf.DataFrame) -> (cf.DataFrame, cf.DataFrame, cf.Series, cf.Series, cupy.ndarray, cf.DataFrame): + """Process data for training/inference operation + + Parameters + ---------- + training_data : cudf.DataFrame + training data + test_data : cudf.DataFrame + test/validation data + + Returns + ------- + tuple + tuple of (training_data, test_data, train_index, test_index, label, combined data) + """ + + train_size = training_data.shape[0] + cdf = cf.concat([training_data, test_data], axis=0) + labels = cdf['fraud_label'].values + + # Drop non-feature columns + cdf.drop(['fraud_label', 'index'], inplace=True, axis=1) + + # Create index of node features + cdf.reset_index(inplace=True) + meta_cols = ['client_node', 'merchant_node'] + for col in meta_cols: + cdf[col] = cf.CategoricalIndex(cdf[col]).codes + + train_data, test_data, train_index, test_index, all_data = (cdf.iloc[:train_size, :], + cdf.iloc[train_size:, :], + cdf['index'][:train_size], + cdf['index'][train_size:], + cdf) + return train_data, test_data, train_index, test_index, labels, all_data diff --git a/examples/gnn_fraud_detection_pipeline/validation.csv b/examples/gnn_fraud_detection_pipeline/validation.csv index 351ecfda8e..82ea9e6db8 100644 --- a/examples/gnn_fraud_detection_pipeline/validation.csv +++ b/examples/gnn_fraud_detection_pipeline/validation.csv @@ -5,262 +5,262 @@ index,1000,1001,client_node,merchant_node,1004,1005,1006,1007,1008,1009,1010,101 756,130.0,1,58567,93773,100486,1,1,7,27,10,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 757,1039.87,0,86378,91782,100499,1,1,14,44,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 758,130.0,1,60551,92009,100486,1,1,8,38,31,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -760,429.91,0,53182,91831,100510,1,1,2,42,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -762,19.5,1,87501,91775,100519,1,1,15,0,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -763,9100.0,0,64035,94642,100757,1,1,9,41,34,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -764,30.42,0,57394,91775,100607,1,1,6,28,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -765,909.87,1,66562,95316,100542,1,1,10,15,25,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -766,25.74,1,74296,91784,100484,1,1,11,57,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -768,191.62,1,70354,96216,100483,1,1,11,6,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -769,116.87,1,77701,91790,100482,1,1,12,41,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -770,883.35,1,73321,92786,100513,1,1,11,44,53,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -771,3887.0,1,84030,91775,100557,1,1,14,9,50,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -774,3627.0,1,72337,96676,100513,1,1,11,31,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -775,65.0,0,58817,93847,100541,1,1,7,41,28,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -776,1974.96,1,80048,91809,100513,1,1,14,21,30,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -777,64.22,1,81603,91808,100510,1,1,13,59,26,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -778,2022.15,1,75515,92001,100486,1,1,12,12,45,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -779,1627.34,1,83576,92326,100500,1,1,14,3,54,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -781,624.0,0,61313,91822,100521,1,1,8,54,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -782,2635.49,1,72782,95457,100561,1,1,11,37,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -783,94.38,1,58744,93845,100491,1,1,8,9,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 -784,1287.0,1,80106,97461,100694,1,1,13,27,54,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -785,571.74,1,70713,96293,100616,1,1,11,10,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -786,416.0,1,58614,93787,100591,1,1,7,29,27,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -787,20.8,1,64221,91775,100607,1,1,9,45,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -788,28457.0,1,88461,99817,100516,1,1,15,15,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -789,1169.74,1,68334,91808,100510,1,1,11,15,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -790,21959.86,1,90072,91819,100521,1,1,15,38,26,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -791,416.0,1,55080,91784,100484,1,1,4,14,42,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -792,838.5,1,74184,94229,100630,1,1,11,55,44,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -794,13.0,1,62094,94973,100732,1,1,9,53,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -795,156.13,1,50554,91986,100482,1,1,1,29,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -796,958.49,1,58762,93794,100565,1,1,7,43,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 -797,1975.48,1,75104,91795,100495,1,1,12,7,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -798,194.35,1,54501,93074,100547,1,1,4,3,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -799,13.0,1,54796,93098,100513,1,1,4,6,26,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 -800,93.08,0,79196,91784,100484,1,1,13,9,56,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -801,240.5,1,84379,97834,100482,1,1,14,15,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -802,1162.2,0,85469,91785,100501,1,1,14,31,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -803,24.05,1,88134,91782,100499,1,1,15,9,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -804,207.35,1,56081,93076,100547,1,1,5,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -805,23.27,0,61038,91760,100484,1,1,8,48,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -806,19.5,1,67007,91795,100498,1,1,10,21,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -807,1341.6,1,86167,94605,100487,1,1,14,41,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -808,467.35,1,64596,94508,100483,1,1,9,50,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -809,22.62,0,60565,91775,100607,1,1,8,39,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -811,168.87,1,80994,91831,100510,1,1,14,15,41,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -812,46.41,1,83696,91760,100484,1,1,14,5,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -813,239.2,1,54131,91899,100532,1,1,3,47,54,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -814,86.32,1,59587,94020,100505,1,1,8,10,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -815,706.94,1,64284,92749,100511,1,1,9,46,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -816,9.62,1,58554,91832,100516,1,1,7,26,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -817,1039.87,1,83593,91977,100490,1,1,14,3,59,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -819,2053.74,0,52471,92526,100514,1,1,2,5,34,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1 -820,38.870000000000005,1,65548,92238,100489,1,1,10,3,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -823,247.13,1,74876,97172,100736,1,1,12,4,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -824,130.0,1,82108,92331,100486,1,1,13,41,34,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -827,507.0,1,74730,92093,100517,1,1,12,2,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -828,6896.5,1,68692,95834,100509,1,1,10,44,54,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -829,11.57,1,53309,91760,100484,1,1,2,51,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -830,124.28,1,55550,91775,100516,1,1,4,41,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -832,3052.4,1,68248,95277,100546,1,1,10,38,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -833,360.75,1,67712,91770,100505,1,1,10,32,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -836,1984.45,1,85102,97643,100575,1,1,14,25,42,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -837,416.0,1,57789,91760,100484,1,1,6,53,56,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -840,1877.2,1,76355,92140,100492,1,1,12,23,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -841,228.02,1,59439,91800,100483,1,1,8,6,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -842,13.0,0,74482,91911,100516,1,1,14,41,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 -843,194.74,1,65478,91760,100484,1,1,10,2,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -844,14.95,0,63010,91791,100607,1,1,9,24,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -845,13.0,1,86500,91832,100516,1,1,14,46,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -846,4743.7,1,51458,92757,100659,1,1,2,44,47,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 -847,392.47,1,82337,91782,100499,1,1,13,44,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -848,844.8699999999999,0,66100,91977,100490,1,1,10,9,59,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -849,16.77,1,73719,91760,100484,1,1,11,49,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -850,13.0,1,55810,92819,100488,1,1,4,56,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -852,21.58,1,63478,94716,100765,1,1,9,32,33,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -853,1045.2,1,52451,91785,100501,1,1,2,5,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -854,2480.4,1,50941,92107,100493,1,1,1,19,23,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -855,416.0,1,55638,91760,100484,1,1,4,45,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -856,2197.0,1,67622,92434,100482,1,1,10,31,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -857,129.87,0,51801,91768,100490,1,1,1,41,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -858,173.94,1,68006,95654,100637,1,1,10,35,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -859,468.0,1,89383,91795,100505,1,1,15,36,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -860,714.87,1,77577,92064,100566,1,1,12,44,46,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -861,247.0,0,75475,97315,100559,1,1,12,12,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 -862,630.24,1,72413,93782,100569,1,1,11,32,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -863,437.84,1,60100,91818,100499,1,1,8,55,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 -864,221.0,1,67454,91795,100498,1,1,10,28,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -865,831.48,1,67795,94593,100567,1,1,10,33,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -866,38.61,1,55820,92992,100499,1,1,4,57,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -867,192.92,1,73926,96988,100530,1,1,11,52,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -868,22352.33,1,67958,95644,100560,1,1,10,35,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -869,208.0,1,56325,91760,100484,1,1,5,16,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -870,512.72,1,85722,92513,100689,1,1,14,40,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -871,455.91,0,51206,92182,100510,1,1,1,25,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -872,559.0,1,87976,99711,100574,1,1,15,7,43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -873,25.74,0,75930,91760,100484,1,1,12,17,55,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -874,13.39,1,59728,91791,100607,1,1,8,15,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -875,1022.06,1,63125,91822,100521,1,1,9,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -876,974.87,0,67751,93924,100490,1,1,10,32,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -877,116.87,1,54089,91784,100484,1,1,3,45,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -878,9.62,1,71264,91832,100516,1,1,11,17,54,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -879,2454.14,1,55711,93254,100490,1,1,4,49,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -881,454.87,1,86975,91954,100570,1,1,14,52,55,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -883,213.98,1,73878,91771,100487,1,1,11,51,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -884,25.74,1,69023,91760,100484,1,1,10,49,23,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -886,62.27,1,58744,93845,100491,1,1,7,58,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 -887,780.0,1,71757,91968,100505,1,1,11,25,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -889,311.87,1,63466,94624,100601,1,1,9,32,21,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -890,7695.48,1,88931,91772,100492,1,1,15,22,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -891,13.0,1,60100,94278,100491,1,1,8,53,55,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 -893,390.0,1,87968,96645,100580,1,1,15,11,50,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -894,64.09,1,90687,93231,100519,1,1,15,47,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -895,129.35,1,87774,92860,100526,1,1,15,4,23,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -896,678.34,1,74867,97167,100487,1,1,12,4,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -897,129.35,1,69603,92064,100566,1,1,10,56,51,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -899,58.370000000000005,1,72355,91784,100484,1,1,11,32,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -900,10239.97,1,66978,95413,100684,1,1,10,20,56,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -901,1676.35,1,50728,92038,100523,1,1,1,14,54,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -902,12.87,1,70500,91784,100484,1,1,11,8,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -903,195.0,1,87450,91864,100526,1,1,15,0,8,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -904,130.0,0,51207,91908,100516,1,1,1,25,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -905,448.5,1,73868,92206,100492,1,1,11,51,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -906,23.27,1,86275,91790,100482,1,1,14,45,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -907,958.49,1,58762,93794,100565,1,1,7,51,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 -908,2184.0,1,61786,95200,100524,1,1,10,6,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -909,552.5,1,63504,91795,100498,1,1,9,32,56,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -911,207.22,1,57215,91770,100557,1,1,6,19,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -913,266.5,1,84542,91770,100516,1,1,14,45,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -914,3446.04,0,76035,92437,100578,1,1,12,19,10,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -917,519.74,1,80389,91771,100487,1,1,13,16,49,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -918,2337.4,1,89191,93973,100565,1,1,15,26,20,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -919,23.14,1,78255,91784,100484,1,1,12,48,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -920,219.05,1,67116,92303,100498,1,1,10,23,15,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -921,1448.98,0,70853,92898,100629,1,1,11,12,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -923,23.27,1,59099,91780,100482,1,1,15,13,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -924,0.0,0,50057,91799,100482,1,1,10,13,50,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -925,415.74,1,89405,91954,100570,1,1,15,29,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -927,25.74,1,67447,91760,100484,1,1,10,28,36,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -929,385.45,1,64646,94801,100483,1,1,9,51,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -930,134.94,1,79091,91808,100510,1,1,13,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -932,221.0,1,76307,91775,100519,1,1,13,0,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -934,533.0,1,91553,91795,100628,1,1,15,59,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -935,23.27,1,70766,91760,100484,1,1,11,11,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -936,416.0,1,56596,91784,100484,1,1,5,33,49,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -937,3204.5,1,60535,94185,100524,1,1,8,38,5,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -938,162.5,1,77906,91795,100516,1,1,12,43,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -939,13.0,1,72738,96762,100492,1,1,11,37,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -940,13.0,1,58744,93889,100516,1,1,7,49,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 -941,495.1700000000001,1,50053,91781,100493,1,1,1,0,55,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -942,0.0,1,74537,93470,100501,1,1,12,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 -943,1027.0,1,79095,91800,100519,1,1,13,0,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -945,7088.9,1,89008,97643,100575,1,1,15,23,25,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -946,381.03,1,90891,91958,100516,1,1,15,50,9,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -947,130.0,1,86524,91770,100625,1,1,14,46,30,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -948,51.870000000000005,1,53067,92626,100519,1,1,2,36,45,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -949,279.5,1,70171,96167,100525,1,1,11,4,20,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -950,1547.0,1,70554,98831,100524,1,1,14,9,39,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -952,1032.07,1,57683,92182,100510,1,1,6,45,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -954,28.340000000000003,1,78178,97901,100574,1,1,12,47,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -955,22854.0,0,56539,91886,100505,1,1,5,30,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -956,98.93,1,80966,91795,100516,1,1,13,24,51,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -957,87.23,1,57498,91791,100607,1,1,6,31,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -958,129.87,1,61637,91774,100494,1,1,9,0,32,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -959,0.0,0,50057,91799,100482,1,1,3,17,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -960,240.5,1,86814,97834,100482,1,1,14,50,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -961,13.0,0,79731,91911,100516,1,1,13,7,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -963,135.20000000000002,1,66758,91831,100510,1,1,10,18,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -964,103.35,1,75193,96453,100516,1,1,12,8,51,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -965,13.0,0,51458,92125,100492,1,1,4,55,54,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 -966,414.57,1,64548,94762,100487,1,1,9,49,49,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -967,1503.84,0,78699,99745,100483,1,1,15,10,46,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 -968,195.0,1,79647,91962,100506,1,1,13,6,27,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -969,416.0,1,54234,91784,100484,1,1,3,54,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -970,38.870000000000005,1,66841,91775,100519,1,1,13,47,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -971,62.00999999999999,0,53602,91795,100557,1,1,3,9,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -972,11.57,1,51034,91780,100482,1,1,15,27,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -973,61.1,1,64012,91770,100670,1,1,11,15,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -974,149.5,1,65734,92019,100490,1,1,10,6,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -975,41.34,1,63314,91775,100607,1,1,9,30,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -976,639.47,1,67902,95631,100665,1,1,10,34,28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -977,129.87,1,75427,91831,100510,1,1,12,11,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -978,1776.84,1,61681,94366,100488,1,1,9,1,12,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -980,259.74,1,90031,91954,100570,1,1,15,37,43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -981,487.5,1,60100,91818,100499,1,1,8,44,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 -982,2990.0,1,64811,94663,100524,1,1,9,53,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -983,2038.4,1,75550,92632,100589,1,1,14,56,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -986,638.56,1,77638,91764,100487,1,1,12,40,41,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -987,12.87,1,59730,91780,100482,1,1,8,18,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -988,13.0,1,51039,91950,100565,1,1,1,21,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -990,2158.0,1,84997,93733,100695,1,1,14,24,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -991,15600.0,1,89401,91770,100498,1,1,15,29,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -992,6708.0,1,62483,94523,100762,1,1,9,15,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -994,360.75,1,61691,91775,100495,1,1,9,1,21,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -997,260.0,1,73313,95307,100492,1,1,11,44,46,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -998,234.0,1,81305,91864,100526,1,1,13,29,49,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1000,0.0,0,50057,91799,100482,1,1,13,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1001,715.0,1,73953,96993,100492,1,1,11,52,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1003,416.0,1,57898,91784,100484,1,1,7,3,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1004,325.0,1,87678,92019,100490,1,1,15,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1005,122.46,1,55954,91760,100484,1,1,5,1,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1007,7425.08,1,52891,93437,100524,1,1,5,44,26,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -1008,11.57,1,63822,94602,100486,1,1,9,39,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1009,347.49,1,87071,92010,100493,1,1,14,54,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1010,209.3,1,84911,95928,100483,1,1,14,22,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1011,285.35,1,83330,91808,100510,1,1,13,59,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1012,358.67,0,81049,92234,100483,1,1,13,29,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1013,221.91,1,87940,91781,100493,1,1,15,7,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1015,20182.24,0,91449,100411,100515,1,1,15,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1016,9.62,1,51924,91997,100519,1,1,1,45,19,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1017,220.87,1,88914,91770,100532,1,1,15,21,41,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1018,254.15,1,61534,94332,100498,1,1,8,59,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -1019,389.87,1,86773,91795,100489,1,1,14,49,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1020,714.87,1,51307,92064,100566,1,1,1,27,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1021,596.6999999999999,1,71939,92379,100482,1,1,11,26,36,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1022,219.7,1,68511,95788,100504,1,1,10,42,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1023,2663.7000000000003,1,68036,93850,100520,1,1,10,36,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1024,25.74,0,69219,91784,100484,1,1,10,55,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1025,844.8699999999999,0,66920,91977,100490,1,1,10,20,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1026,317.85,1,82063,91764,100487,1,1,13,40,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1028,0.0,0,50057,91799,100482,1,1,13,21,50,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1029,10039.77,1,52954,92140,100492,1,1,2,29,59,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1030,1794.0,0,72983,91868,100537,1,1,11,40,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1031,5043.87,1,62120,94489,100492,1,1,9,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 -1032,13436.8,1,76956,97647,100580,1,1,12,31,52,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1033,239.59,1,71377,95293,100593,1,1,11,19,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -1034,757.77,1,76731,95327,100511,1,1,12,28,32,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1035,245.7,1,50052,91781,100493,1,1,1,0,55,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1036,3900.0,1,85884,91868,100537,1,1,14,37,35,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1037,722.41,1,82119,98676,100533,1,1,13,41,42,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1038,47.97,1,65443,91782,100499,1,1,10,2,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1039,130.0,1,63308,93334,100530,1,1,9,30,12,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1040,416.0,1,56421,91760,100484,1,1,5,22,19,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1042,287.43,1,72853,93871,100511,1,1,11,38,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -1043,714.35,1,78410,91770,100516,1,1,15,52,46,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1044,156.0,0,76619,91928,100566,1,1,12,27,6,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1046,195.0,1,50488,91962,100506,1,1,1,10,10,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1047,844.35,1,52589,92530,100490,1,1,2,10,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1048,935.48,1,78661,91979,100603,1,1,12,54,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1049,110.37,1,52030,91775,100499,1,1,2,3,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1050,239.59,1,54049,92998,100693,1,1,3,42,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -1053,51.870000000000005,1,58744,93845,100491,1,1,8,23,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 -1054,192.92,1,60100,94278,100491,1,1,9,42,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 -1055,168.87,1,60525,91760,100484,1,1,8,37,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1056,518.6999999999999,0,56904,93474,100566,1,1,6,9,21,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 -1057,1429.35,1,79614,91795,100502,1,1,13,6,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1058,6493.37,1,69229,91772,100492,1,1,10,52,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1060,1156.22,1,78044,91764,100487,1,1,12,45,43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1062,13.0,1,80135,97041,100695,1,1,13,15,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 -1063,584.87,1,56718,91784,100484,1,1,5,43,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1064,194.35,1,54478,93074,100547,1,1,4,3,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1065,955.5,1,74506,94229,100630,1,1,11,59,49,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1066,1690.0,1,73740,96957,100570,1,1,11,50,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1067,16.77,1,60553,91760,100484,1,1,8,38,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1068,389.87,1,63454,91928,100566,1,1,9,32,11,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1069,19.37,1,89320,91782,100499,1,1,15,31,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1070,1341.86,1,51505,91800,100516,1,1,1,31,59,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 -1071,46.54,1,65324,91760,100484,1,1,10,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1072,389.35,0,53482,91928,100566,1,1,3,3,12,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1075,2210.0,1,63938,94811,100768,1,1,9,39,49,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +759,429.91,0,53182,91831,100510,1,1,2,42,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +760,19.5,1,87501,91775,100519,1,1,15,0,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +761,9100.0,0,64035,94642,100757,1,1,9,41,34,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +762,30.42,0,57394,91775,100607,1,1,6,28,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +763,909.87,1,66562,95316,100542,1,1,10,15,25,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +764,25.74,1,74296,91784,100484,1,1,11,57,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +765,191.62,1,70354,96216,100483,1,1,11,6,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +766,116.87,1,77701,91790,100482,1,1,12,41,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +767,883.35,1,73321,92786,100513,1,1,11,44,53,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +768,3887.0,1,84030,91775,100557,1,1,14,9,50,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +769,3627.0,1,72337,96676,100513,1,1,11,31,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +770,65.0,0,58817,93847,100541,1,1,7,41,28,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +771,1974.96,1,80048,91809,100513,1,1,14,21,30,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +772,64.22,1,81603,91808,100510,1,1,13,59,26,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +773,2022.15,1,75515,92001,100486,1,1,12,12,45,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +774,1627.34,1,83576,92326,100500,1,1,14,3,54,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +775,624.0,0,61313,91822,100521,1,1,8,54,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +776,2635.49,1,72782,95457,100561,1,1,11,37,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +777,94.38,1,58744,93845,100491,1,1,8,9,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 +778,1287.0,1,80106,97461,100694,1,1,13,27,54,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +779,571.74,1,70713,96293,100616,1,1,11,10,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +780,416.0,1,58614,93787,100591,1,1,7,29,27,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +781,20.8,1,64221,91775,100607,1,1,9,45,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +782,28457.0,1,88461,99817,100516,1,1,15,15,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +783,1169.74,1,68334,91808,100510,1,1,11,15,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +784,21959.86,1,90072,91819,100521,1,1,15,38,26,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +785,416.0,1,55080,91784,100484,1,1,4,14,42,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +786,838.5,1,74184,94229,100630,1,1,11,55,44,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +787,13.0,1,62094,94973,100732,1,1,9,53,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +788,156.13,1,50554,91986,100482,1,1,1,29,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +789,958.49,1,58762,93794,100565,1,1,7,43,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 +790,1975.48,1,75104,91795,100495,1,1,12,7,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +791,194.35,1,54501,93074,100547,1,1,4,3,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +792,13.0,1,54796,93098,100513,1,1,4,6,26,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +793,93.08,0,79196,91784,100484,1,1,13,9,56,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +794,240.5,1,84379,97834,100482,1,1,14,15,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +795,1162.2,0,85469,91785,100501,1,1,14,31,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +796,24.05,1,88134,91782,100499,1,1,15,9,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +797,207.35,1,56081,93076,100547,1,1,5,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +798,23.27,0,61038,91760,100484,1,1,8,48,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +799,19.5,1,67007,91795,100498,1,1,10,21,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +800,1341.6,1,86167,94605,100487,1,1,14,41,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +801,467.35,1,64596,94508,100483,1,1,9,50,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +802,22.62,0,60565,91775,100607,1,1,8,39,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +803,168.87,1,80994,91831,100510,1,1,14,15,41,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +804,46.41,1,83696,91760,100484,1,1,14,5,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +805,239.2,1,54131,91899,100532,1,1,3,47,54,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +806,86.32,1,59587,94020,100505,1,1,8,10,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +807,706.94,1,64284,92749,100511,1,1,9,46,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +808,9.62,1,58554,91832,100516,1,1,7,26,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +809,1039.87,1,83593,91977,100490,1,1,14,3,59,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +810,2053.74,0,52471,92526,100514,1,1,2,5,34,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1 +811,38.870000000000005,1,65548,92238,100489,1,1,10,3,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +812,247.13,1,74876,97172,100736,1,1,12,4,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +813,130.0,1,82108,92331,100486,1,1,13,41,34,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +814,507.0,1,74730,92093,100517,1,1,12,2,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +815,6896.5,1,68692,95834,100509,1,1,10,44,54,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +816,11.57,1,53309,91760,100484,1,1,2,51,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +817,124.28,1,55550,91775,100516,1,1,4,41,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +818,3052.4,1,68248,95277,100546,1,1,10,38,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +819,360.75,1,67712,91770,100505,1,1,10,32,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +820,1984.45,1,85102,97643,100575,1,1,14,25,42,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +821,416.0,1,57789,91760,100484,1,1,6,53,56,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +822,1877.2,1,76355,92140,100492,1,1,12,23,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +823,228.02,1,59439,91800,100483,1,1,8,6,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +824,13.0,0,74482,91911,100516,1,1,14,41,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +825,194.74,1,65478,91760,100484,1,1,10,2,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +826,14.95,0,63010,91791,100607,1,1,9,24,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +827,13.0,1,86500,91832,100516,1,1,14,46,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +828,4743.7,1,51458,92757,100659,1,1,2,44,47,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +829,392.47,1,82337,91782,100499,1,1,13,44,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +830,844.8699999999999,0,66100,91977,100490,1,1,10,9,59,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +831,16.77,1,73719,91760,100484,1,1,11,49,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +832,13.0,1,55810,92819,100488,1,1,4,56,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +833,21.58,1,63478,94716,100765,1,1,9,32,33,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +834,1045.2,1,52451,91785,100501,1,1,2,5,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +835,2480.4,1,50941,92107,100493,1,1,1,19,23,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +836,416.0,1,55638,91760,100484,1,1,4,45,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +837,2197.0,1,67622,92434,100482,1,1,10,31,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +838,129.87,0,51801,91768,100490,1,1,1,41,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +839,173.94,1,68006,95654,100637,1,1,10,35,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +840,468.0,1,89383,91795,100505,1,1,15,36,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +841,714.87,1,77577,92064,100566,1,1,12,44,46,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +842,247.0,0,75475,97315,100559,1,1,12,12,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +843,630.24,1,72413,93782,100569,1,1,11,32,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +844,437.84,1,60100,91818,100499,1,1,8,55,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +845,221.0,1,67454,91795,100498,1,1,10,28,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +846,831.48,1,67795,94593,100567,1,1,10,33,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +847,38.61,1,55820,92992,100499,1,1,4,57,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +848,192.92,1,73926,96988,100530,1,1,11,52,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +849,22352.33,1,67958,95644,100560,1,1,10,35,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +850,208.0,1,56325,91760,100484,1,1,5,16,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +851,512.72,1,85722,92513,100689,1,1,14,40,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +852,455.91,0,51206,92182,100510,1,1,1,25,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +853,559.0,1,87976,99711,100574,1,1,15,7,43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +854,25.74,0,75930,91760,100484,1,1,12,17,55,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +855,13.39,1,59728,91791,100607,1,1,8,15,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +856,1022.06,1,63125,91822,100521,1,1,9,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +857,974.87,0,67751,93924,100490,1,1,10,32,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +858,116.87,1,54089,91784,100484,1,1,3,45,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +859,9.62,1,71264,91832,100516,1,1,11,17,54,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +860,2454.14,1,55711,93254,100490,1,1,4,49,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +861,454.87,1,86975,91954,100570,1,1,14,52,55,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +862,213.98,1,73878,91771,100487,1,1,11,51,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +863,25.74,1,69023,91760,100484,1,1,10,49,23,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +864,62.27,1,58744,93845,100491,1,1,7,58,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 +865,780.0,1,71757,91968,100505,1,1,11,25,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +866,311.87,1,63466,94624,100601,1,1,9,32,21,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +867,7695.48,1,88931,91772,100492,1,1,15,22,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +868,13.0,1,60100,94278,100491,1,1,8,53,55,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 +869,390.0,1,87968,96645,100580,1,1,15,11,50,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +870,64.09,1,90687,93231,100519,1,1,15,47,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +871,129.35,1,87774,92860,100526,1,1,15,4,23,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +872,678.34,1,74867,97167,100487,1,1,12,4,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +873,129.35,1,69603,92064,100566,1,1,10,56,51,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +874,58.370000000000005,1,72355,91784,100484,1,1,11,32,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +875,10239.97,1,66978,95413,100684,1,1,10,20,56,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +876,1676.35,1,50728,92038,100523,1,1,1,14,54,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +877,12.87,1,70500,91784,100484,1,1,11,8,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +878,195.0,1,87450,91864,100526,1,1,15,0,8,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +879,130.0,0,51207,91908,100516,1,1,1,25,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +880,448.5,1,73868,92206,100492,1,1,11,51,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +881,23.27,1,86275,91790,100482,1,1,14,45,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +882,958.49,1,58762,93794,100565,1,1,7,51,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 +883,2184.0,1,61786,95200,100524,1,1,10,6,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +884,552.5,1,63504,91795,100498,1,1,9,32,56,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +885,207.22,1,57215,91770,100557,1,1,6,19,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +886,266.5,1,84542,91770,100516,1,1,14,45,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +887,3446.04,0,76035,92437,100578,1,1,12,19,10,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +888,519.74,1,80389,91771,100487,1,1,13,16,49,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +889,2337.4,1,89191,93973,100565,1,1,15,26,20,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +890,23.14,1,78255,91784,100484,1,1,12,48,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +891,219.05,1,67116,92303,100498,1,1,10,23,15,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +892,1448.98,0,70853,92898,100629,1,1,11,12,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +893,23.27,1,59099,91780,100482,1,1,15,13,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +894,0.0,0,50057,91799,100482,1,1,10,13,50,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +895,415.74,1,89405,91954,100570,1,1,15,29,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +896,25.74,1,67447,91760,100484,1,1,10,28,36,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +897,385.45,1,64646,94801,100483,1,1,9,51,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +898,134.94,1,79091,91808,100510,1,1,13,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +899,221.0,1,76307,91775,100519,1,1,13,0,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +900,533.0,1,91553,91795,100628,1,1,15,59,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +901,23.27,1,70766,91760,100484,1,1,11,11,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +902,416.0,1,56596,91784,100484,1,1,5,33,49,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +903,3204.5,1,60535,94185,100524,1,1,8,38,5,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +904,162.5,1,77906,91795,100516,1,1,12,43,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +905,13.0,1,72738,96762,100492,1,1,11,37,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +906,13.0,1,58744,93889,100516,1,1,7,49,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 +907,495.1700000000001,1,50053,91781,100493,1,1,1,0,55,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +908,0.0,1,74537,93470,100501,1,1,12,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 +909,1027.0,1,79095,91800,100519,1,1,13,0,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +910,7088.9,1,89008,97643,100575,1,1,15,23,25,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +911,381.03,1,90891,91958,100516,1,1,15,50,9,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +912,130.0,1,86524,91770,100625,1,1,14,46,30,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +913,51.870000000000005,1,53067,92626,100519,1,1,2,36,45,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +914,279.5,1,70171,96167,100525,1,1,11,4,20,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +915,1547.0,1,70554,98831,100524,1,1,14,9,39,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +916,1032.07,1,57683,92182,100510,1,1,6,45,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +917,28.340000000000003,1,78178,97901,100574,1,1,12,47,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +918,22854.0,0,56539,91886,100505,1,1,5,30,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +919,98.93,1,80966,91795,100516,1,1,13,24,51,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +920,87.23,1,57498,91791,100607,1,1,6,31,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +921,129.87,1,61637,91774,100494,1,1,9,0,32,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +922,0.0,0,50057,91799,100482,1,1,3,17,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +923,240.5,1,86814,97834,100482,1,1,14,50,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +924,13.0,0,79731,91911,100516,1,1,13,7,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +925,135.20000000000002,1,66758,91831,100510,1,1,10,18,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +926,103.35,1,75193,96453,100516,1,1,12,8,51,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +927,13.0,0,51458,92125,100492,1,1,4,55,54,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +928,414.57,1,64548,94762,100487,1,1,9,49,49,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +929,1503.84,0,78699,99745,100483,1,1,15,10,46,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +930,195.0,1,79647,91962,100506,1,1,13,6,27,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +931,416.0,1,54234,91784,100484,1,1,3,54,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +932,38.870000000000005,1,66841,91775,100519,1,1,13,47,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +933,62.00999999999999,0,53602,91795,100557,1,1,3,9,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +934,11.57,1,51034,91780,100482,1,1,15,27,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +935,61.1,1,64012,91770,100670,1,1,11,15,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +936,149.5,1,65734,92019,100490,1,1,10,6,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +937,41.34,1,63314,91775,100607,1,1,9,30,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +938,639.47,1,67902,95631,100665,1,1,10,34,28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +939,129.87,1,75427,91831,100510,1,1,12,11,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +940,1776.84,1,61681,94366,100488,1,1,9,1,12,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +941,259.74,1,90031,91954,100570,1,1,15,37,43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +942,487.5,1,60100,91818,100499,1,1,8,44,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +943,2990.0,1,64811,94663,100524,1,1,9,53,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +944,2038.4,1,75550,92632,100589,1,1,14,56,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +945,638.56,1,77638,91764,100487,1,1,12,40,41,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +946,12.87,1,59730,91780,100482,1,1,8,18,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +947,13.0,1,51039,91950,100565,1,1,1,21,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +948,2158.0,1,84997,93733,100695,1,1,14,24,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +949,15600.0,1,89401,91770,100498,1,1,15,29,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +950,6708.0,1,62483,94523,100762,1,1,9,15,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +951,360.75,1,61691,91775,100495,1,1,9,1,21,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +952,260.0,1,73313,95307,100492,1,1,11,44,46,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +953,234.0,1,81305,91864,100526,1,1,13,29,49,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +954,0.0,0,50057,91799,100482,1,1,13,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +955,715.0,1,73953,96993,100492,1,1,11,52,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +956,416.0,1,57898,91784,100484,1,1,7,3,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +957,325.0,1,87678,92019,100490,1,1,15,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +958,122.46,1,55954,91760,100484,1,1,5,1,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +959,7425.08,1,52891,93437,100524,1,1,5,44,26,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +960,11.57,1,63822,94602,100486,1,1,9,39,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +961,347.49,1,87071,92010,100493,1,1,14,54,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +962,209.3,1,84911,95928,100483,1,1,14,22,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +963,285.35,1,83330,91808,100510,1,1,13,59,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +964,358.67,0,81049,92234,100483,1,1,13,29,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +965,221.91,1,87940,91781,100493,1,1,15,7,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +966,20182.24,0,91449,100411,100515,1,1,15,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +967,9.62,1,51924,91997,100519,1,1,1,45,19,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +968,220.87,1,88914,91770,100532,1,1,15,21,41,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +969,254.15,1,61534,94332,100498,1,1,8,59,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +970,389.87,1,86773,91795,100489,1,1,14,49,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +971,714.87,1,51307,92064,100566,1,1,1,27,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +972,596.6999999999999,1,71939,92379,100482,1,1,11,26,36,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +973,219.7,1,68511,95788,100504,1,1,10,42,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +974,2663.7000000000003,1,68036,93850,100520,1,1,10,36,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +975,25.74,0,69219,91784,100484,1,1,10,55,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +976,844.8699999999999,0,66920,91977,100490,1,1,10,20,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +977,317.85,1,82063,91764,100487,1,1,13,40,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +978,0.0,0,50057,91799,100482,1,1,13,21,50,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +979,10039.77,1,52954,92140,100492,1,1,2,29,59,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +980,1794.0,0,72983,91868,100537,1,1,11,40,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +981,5043.87,1,62120,94489,100492,1,1,9,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +982,13436.8,1,76956,97647,100580,1,1,12,31,52,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +983,239.59,1,71377,95293,100593,1,1,11,19,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +984,757.77,1,76731,95327,100511,1,1,12,28,32,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +985,245.7,1,50052,91781,100493,1,1,1,0,55,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +986,3900.0,1,85884,91868,100537,1,1,14,37,35,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +987,722.41,1,82119,98676,100533,1,1,13,41,42,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +988,47.97,1,65443,91782,100499,1,1,10,2,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +989,130.0,1,63308,93334,100530,1,1,9,30,12,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +990,416.0,1,56421,91760,100484,1,1,5,22,19,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +991,287.43,1,72853,93871,100511,1,1,11,38,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +992,714.35,1,78410,91770,100516,1,1,15,52,46,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +993,156.0,0,76619,91928,100566,1,1,12,27,6,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +994,195.0,1,50488,91962,100506,1,1,1,10,10,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +995,844.35,1,52589,92530,100490,1,1,2,10,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +996,935.48,1,78661,91979,100603,1,1,12,54,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +997,110.37,1,52030,91775,100499,1,1,2,3,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +998,239.59,1,54049,92998,100693,1,1,3,42,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +999,51.870000000000005,1,58744,93845,100491,1,1,8,23,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 +1000,192.92,1,60100,94278,100491,1,1,9,42,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1 +1001,168.87,1,60525,91760,100484,1,1,8,37,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1002,518.6999999999999,0,56904,93474,100566,1,1,6,9,21,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +1003,1429.35,1,79614,91795,100502,1,1,13,6,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1004,6493.37,1,69229,91772,100492,1,1,10,52,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1005,1156.22,1,78044,91764,100487,1,1,12,45,43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1006,13.0,1,80135,97041,100695,1,1,13,15,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +1007,584.87,1,56718,91784,100484,1,1,5,43,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1008,194.35,1,54478,93074,100547,1,1,4,3,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1009,955.5,1,74506,94229,100630,1,1,11,59,49,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1010,1690.0,1,73740,96957,100570,1,1,11,50,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1011,16.77,1,60553,91760,100484,1,1,8,38,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1012,389.87,1,63454,91928,100566,1,1,9,32,11,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1013,19.37,1,89320,91782,100499,1,1,15,31,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1014,1341.86,1,51505,91800,100516,1,1,1,31,59,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0 +1015,46.54,1,65324,91760,100484,1,1,10,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1016,389.35,0,53482,91928,100566,1,1,3,3,12,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1017,2210.0,1,63938,94811,100768,1,1,9,39,49,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/models/.gitattributes b/models/.gitattributes index f8f33cdbbc..0acf2cb594 100644 --- a/models/.gitattributes +++ b/models/.gitattributes @@ -6,3 +6,4 @@ log-parsing-models/* filter=lfs diff=lfs merge=lfs -text phishing-models/* filter=lfs diff=lfs merge=lfs -text root-cause-models/* filter=lfs diff=lfs merge=lfs -text sid-models/* filter=lfs diff=lfs merge=lfs -text +fraud-detection-models/ filter=lfs diff=lfs merge=lfs -text diff --git a/models/fraud-detection-models/graph.pkl b/models/fraud-detection-models/graph.pkl new file mode 100644 index 0000000000..6d5706fc4e Binary files /dev/null and b/models/fraud-detection-models/graph.pkl differ diff --git a/models/fraud-detection-models/hinsage-model.pt/keras_metadata.pb b/models/fraud-detection-models/hinsage-model.pt/keras_metadata.pb deleted file mode 100644 index cf4ce6caa6..0000000000 --- a/models/fraud-detection-models/hinsage-model.pt/keras_metadata.pb +++ /dev/null @@ -1,30 +0,0 @@ - -root"_tf_keras_network*{"name": "model_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "class_name": "Functional", "config": {"name": "model_1", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_4"}, "name": "input_4", "inbound_nodes": []}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_5"}, "name": "input_5", "inbound_nodes": []}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}, "name": "input_2", "inbound_nodes": []}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_3"}, "name": "input_3", "inbound_nodes": []}, {"class_name": "Reshape", "config": {"name": "reshape_2", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "name": "reshape_2", "inbound_nodes": [[["input_4", 0, 0, {}]]]}, {"class_name": "Reshape", "config": {"name": "reshape_3", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "name": "reshape_3", "inbound_nodes": [[["input_5", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 1, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_1"}, "name": "input_1", "inbound_nodes": []}, {"class_name": "Reshape", "config": {"name": "reshape", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "name": "reshape", "inbound_nodes": [[["input_2", 0, 0, {}]]]}, {"class_name": "Reshape", "config": {"name": "reshape_1", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "name": "reshape_1", "inbound_nodes": [[["input_3", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_4", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_4", "inbound_nodes": [[["input_2", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_3", "inbound_nodes": [[["reshape_2", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_6", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_6", "inbound_nodes": [[["input_3", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_5", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_5", "inbound_nodes": [[["reshape_3", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_2", "inbound_nodes": [[["input_1", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout", "inbound_nodes": [[["reshape", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_1", "inbound_nodes": [[["reshape_1", 0, 0, {}]]]}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_1", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_1", "inbound_nodes": [[["dropout_4", 0, 0, {}], ["dropout_3", 0, 0, {}]]]}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_2", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_2", "inbound_nodes": [[["dropout_6", 0, 0, {}], ["dropout_5", 0, 0, {}]]]}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator", "inbound_nodes": [[["dropout_2", 0, 0, {}], ["dropout", 0, 0, {}], ["dropout_1", 0, 0, {}]]]}, {"class_name": "Reshape", "config": {"name": "reshape_4", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "name": "reshape_4", "inbound_nodes": [[["mean_hin_aggregator_1", 0, 0, {}]]]}, {"class_name": "Reshape", "config": {"name": "reshape_5", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "name": "reshape_5", "inbound_nodes": [[["mean_hin_aggregator_2", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_9", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_9", "inbound_nodes": [[["mean_hin_aggregator", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_7", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_7", "inbound_nodes": [[["reshape_4", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_8", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_8", "inbound_nodes": [[["reshape_5", 0, 0, {}]]]}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_3", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "linear", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_3", "inbound_nodes": [[["dropout_9", 0, 0, {}], ["dropout_7", 0, 0, {}], ["dropout_8", 0, 0, {}]]]}, {"class_name": "Reshape", "config": {"name": "reshape_6", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [64]}}, "name": "reshape_6", "inbound_nodes": [[["mean_hin_aggregator_3", 0, 0, {}]]]}, {"class_name": "Lambda", "config": {"name": "lambda", "trainable": true, "dtype": "float32", "function": {"class_name": "__tuple__", "items": ["4wEAAAAAAAAAAAAAAAEAAAAEAAAAUwAAAHMOAAAAdABqAXwAZAFkAo0CUwApA07p/////6kB2gRh\neGlzKQLaAUvaDGwyX25vcm1hbGl6ZakB2gF4qQByCAAAAPpQL29wdC9jb25kYS9lbnZzL3JhcGlk\ncy9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RlbGxhcmdyYXBoL2xheWVyL2hpbnNhZ2Uu\ncHnaCDxsYW1iZGE+ZgEAAPMAAAAA\n", null, null]}, "function_type": "lambda", "module": "stellargraph.layer.hinsage", "output_shape": null, "output_shape_type": "raw", "output_shape_module": null, "arguments": {}}, "name": "lambda", "inbound_nodes": [[["reshape_6", 0, 0, {}]]]}], "input_layers": [["input_1", 0, 0], ["input_2", 0, 0], ["input_3", 0, 0], ["input_4", 0, 0], ["input_5", 0, 0]], "output_layers": [["lambda", 0, 0]]}, "shared_object_id": 35, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 1, 111]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}, {"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}, {"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}, {"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}, {"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": [{"class_name": "TensorShape", "items": [null, 1, 111]}, {"class_name": "TensorShape", "items": [null, 2, 1]}, {"class_name": "TensorShape", "items": [null, 2, 1]}, {"class_name": "TensorShape", "items": [null, 64, 111]}, {"class_name": "TensorShape", "items": [null, 64, 111]}], "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 1, 111]}, "float32", "input_1"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 2, 1]}, "float32", "input_2"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 2, 1]}, "float32", "input_3"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 64, 111]}, "float32", "input_4"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 64, 111]}, "float32", "input_5"]}]], {}]}, "save_spec": [{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 1, 111]}, "float32", "input_1"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 2, 1]}, "float32", "input_2"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 2, 1]}, "float32", "input_3"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 64, 111]}, "float32", "input_4"]}, {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 64, 111]}, "float32", "input_5"]}], "keras_version": "2.6.0", "backend": "tensorflow", "model_config": {"class_name": "Functional", "config": {"name": "model_1", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_4"}, "name": "input_4", "inbound_nodes": [], "shared_object_id": 0}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_5"}, "name": "input_5", "inbound_nodes": [], "shared_object_id": 1}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}, "name": "input_2", "inbound_nodes": [], "shared_object_id": 2}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_3"}, "name": "input_3", "inbound_nodes": [], "shared_object_id": 3}, {"class_name": "Reshape", "config": {"name": "reshape_2", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "name": "reshape_2", "inbound_nodes": [[["input_4", 0, 0, {}]]], "shared_object_id": 4}, {"class_name": "Reshape", "config": {"name": "reshape_3", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "name": "reshape_3", "inbound_nodes": [[["input_5", 0, 0, {}]]], "shared_object_id": 5}, {"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 1, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_1"}, "name": "input_1", "inbound_nodes": [], "shared_object_id": 6}, {"class_name": "Reshape", "config": {"name": "reshape", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "name": "reshape", "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 7}, {"class_name": "Reshape", "config": {"name": "reshape_1", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "name": "reshape_1", "inbound_nodes": [[["input_3", 0, 0, {}]]], "shared_object_id": 8}, {"class_name": "Dropout", "config": {"name": "dropout_4", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_4", "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 9}, {"class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_3", "inbound_nodes": [[["reshape_2", 0, 0, {}]]], "shared_object_id": 10}, {"class_name": "Dropout", "config": {"name": "dropout_6", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_6", "inbound_nodes": [[["input_3", 0, 0, {}]]], "shared_object_id": 11}, {"class_name": "Dropout", "config": {"name": "dropout_5", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_5", "inbound_nodes": [[["reshape_3", 0, 0, {}]]], "shared_object_id": 12}, {"class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_2", "inbound_nodes": [[["input_1", 0, 0, {}]]], "shared_object_id": 13}, {"class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout", "inbound_nodes": [[["reshape", 0, 0, {}]]], "shared_object_id": 14}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_1", "inbound_nodes": [[["reshape_1", 0, 0, {}]]], "shared_object_id": 15}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_1", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 16}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 17}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_1", "inbound_nodes": [[["dropout_4", 0, 0, {}], ["dropout_3", 0, 0, {}]]], "shared_object_id": 18}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_2", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 19}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 20}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_2", "inbound_nodes": [[["dropout_6", 0, 0, {}], ["dropout_5", 0, 0, {}]]], "shared_object_id": 21}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 22}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 23}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator", "inbound_nodes": [[["dropout_2", 0, 0, {}], ["dropout", 0, 0, {}], ["dropout_1", 0, 0, {}]]], "shared_object_id": 24}, {"class_name": "Reshape", "config": {"name": "reshape_4", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "name": "reshape_4", "inbound_nodes": [[["mean_hin_aggregator_1", 0, 0, {}]]], "shared_object_id": 25}, {"class_name": "Reshape", "config": {"name": "reshape_5", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "name": "reshape_5", "inbound_nodes": [[["mean_hin_aggregator_2", 0, 0, {}]]], "shared_object_id": 26}, {"class_name": "Dropout", "config": {"name": "dropout_9", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_9", "inbound_nodes": [[["mean_hin_aggregator", 0, 0, {}]]], "shared_object_id": 27}, {"class_name": "Dropout", "config": {"name": "dropout_7", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_7", "inbound_nodes": [[["reshape_4", 0, 0, {}]]], "shared_object_id": 28}, {"class_name": "Dropout", "config": {"name": "dropout_8", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "name": "dropout_8", "inbound_nodes": [[["reshape_5", 0, 0, {}]]], "shared_object_id": 29}, {"class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_3", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "linear", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 30}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 31}, "bias_regularizer": null, "bias_constraint": null}, "name": "mean_hin_aggregator_3", "inbound_nodes": [[["dropout_9", 0, 0, {}], ["dropout_7", 0, 0, {}], ["dropout_8", 0, 0, {}]]], "shared_object_id": 32}, {"class_name": "Reshape", "config": {"name": "reshape_6", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [64]}}, "name": "reshape_6", "inbound_nodes": [[["mean_hin_aggregator_3", 0, 0, {}]]], "shared_object_id": 33}, {"class_name": "Lambda", "config": {"name": "lambda", "trainable": true, "dtype": "float32", "function": {"class_name": "__tuple__", "items": ["4wEAAAAAAAAAAAAAAAEAAAAEAAAAUwAAAHMOAAAAdABqAXwAZAFkAo0CUwApA07p/////6kB2gRh\neGlzKQLaAUvaDGwyX25vcm1hbGl6ZakB2gF4qQByCAAAAPpQL29wdC9jb25kYS9lbnZzL3JhcGlk\ncy9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RlbGxhcmdyYXBoL2xheWVyL2hpbnNhZ2Uu\ncHnaCDxsYW1iZGE+ZgEAAPMAAAAA\n", null, null]}, "function_type": "lambda", "module": "stellargraph.layer.hinsage", "output_shape": null, "output_shape_type": "raw", "output_shape_module": null, "arguments": {}}, "name": "lambda", "inbound_nodes": [[["reshape_6", 0, 0, {}]]], "shared_object_id": 34}], "input_layers": [["input_1", 0, 0], ["input_2", 0, 0], ["input_3", 0, 0], ["input_4", 0, 0], ["input_5", 0, 0]], "output_layers": [["lambda", 0, 0]]}}}2 - root.layer-0"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_4", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_4"}}2 - root.layer-1"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_5", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 64, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_5"}}2 - root.layer-2"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_2", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}}2 - root.layer-3"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_3", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 2, 1]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_3"}}2 - root.layer-4"_tf_keras_layer*{"name": "reshape_2", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_2", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "inbound_nodes": [[["input_4", 0, 0, {}]]], "shared_object_id": 4}2 - root.layer-5"_tf_keras_layer*{"name": "reshape_3", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_3", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [2, 32, 111]}}, "inbound_nodes": [[["input_5", 0, 0, {}]]], "shared_object_id": 5}2 - root.layer-6"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_1", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 1, 111]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 1, 111]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_1"}}2 - root.layer-7"_tf_keras_layer*{"name": "reshape", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 7}2 -  root.layer-8"_tf_keras_layer*{"name": "reshape_1", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_1", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 1]}}, "inbound_nodes": [[["input_3", 0, 0, {}]]], "shared_object_id": 8}2 - - root.layer-9"_tf_keras_layer*{"name": "dropout_4", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_4", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 9}2 -  root.layer-10"_tf_keras_layer*{"name": "dropout_3", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape_2", 0, 0, {}]]], "shared_object_id": 10}2 -  root.layer-11"_tf_keras_layer*{"name": "dropout_6", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_6", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["input_3", 0, 0, {}]]], "shared_object_id": 11}2 -  root.layer-12"_tf_keras_layer*{"name": "dropout_5", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_5", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape_3", 0, 0, {}]]], "shared_object_id": 12}2 - root.layer-13"_tf_keras_layer*{"name": "dropout_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["input_1", 0, 0, {}]]], "shared_object_id": 13}2 - root.layer-14"_tf_keras_layer*{"name": "dropout", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape", 0, 0, {}]]], "shared_object_id": 14}2 - root.layer-15"_tf_keras_layer*{"name": "dropout_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape_1", 0, 0, {}]]], "shared_object_id": 15}2 -root.layer_with_weights-0"_tf_keras_layer*{"name": "mean_hin_aggregator_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_1", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 16}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 17}, "bias_regularizer": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_4", 0, 0, {}], ["dropout_3", 0, 0, {}]]], "shared_object_id": 18, "build_input_shape": [{"class_name": "TensorShape", "items": [null, 2, 1]}, {"class_name": "TensorShape", "items": [null, 2, 32, 111]}]}2 -root.layer_with_weights-1"_tf_keras_layer*{"name": "mean_hin_aggregator_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_2", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 19}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 20}, "bias_regularizer": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_6", 0, 0, {}], ["dropout_5", 0, 0, {}]]], "shared_object_id": 21, "build_input_shape": [{"class_name": "TensorShape", "items": [null, 2, 1]}, {"class_name": "TensorShape", "items": [null, 2, 32, 111]}]}2 -root.layer_with_weights-2"_tf_keras_layer*{"name": "mean_hin_aggregator", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "relu", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 22}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 23}, "bias_regularizer": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_2", 0, 0, {}], ["dropout", 0, 0, {}], ["dropout_1", 0, 0, {}]]], "shared_object_id": 24, "build_input_shape": [{"class_name": "TensorShape", "items": [null, 1, 111]}, {"class_name": "TensorShape", "items": [null, 1, 2, 1]}, {"class_name": "TensorShape", "items": [null, 1, 2, 1]}]}2 - root.layer-19"_tf_keras_layer*{"name": "reshape_4", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_4", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "inbound_nodes": [[["mean_hin_aggregator_1", 0, 0, {}]]], "shared_object_id": 25}2 - root.layer-20"_tf_keras_layer*{"name": "reshape_5", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_5", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [1, 2, 64]}}, "inbound_nodes": [[["mean_hin_aggregator_2", 0, 0, {}]]], "shared_object_id": 26}2 - root.layer-21"_tf_keras_layer*{"name": "dropout_9", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_9", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["mean_hin_aggregator", 0, 0, {}]]], "shared_object_id": 27}2 - root.layer-22"_tf_keras_layer*{"name": "dropout_7", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_7", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape_4", 0, 0, {}]]], "shared_object_id": 28}2 - root.layer-23"_tf_keras_layer*{"name": "dropout_8", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_8", "trainable": true, "dtype": "float32", "rate": 0, "noise_shape": null, "seed": null}, "inbound_nodes": [[["reshape_5", 0, 0, {}]]], "shared_object_id": 29}2 -root.layer_with_weights-3"_tf_keras_layer*{"name": "mean_hin_aggregator_3", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "MeanHinAggregator", "config": {"name": "mean_hin_aggregator_3", "trainable": true, "dtype": "float32", "output_dim": 64, "bias": true, "act": "linear", "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 30}, "kernel_regularizer": null, "kernel_constraint": null, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 31}, "bias_regularizer": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_9", 0, 0, {}], ["dropout_7", 0, 0, {}], ["dropout_8", 0, 0, {}]]], "shared_object_id": 32, "build_input_shape": [{"class_name": "TensorShape", "items": [null, 1, 64]}, {"class_name": "TensorShape", "items": [null, 1, 2, 64]}, {"class_name": "TensorShape", "items": [null, 1, 2, 64]}]}2 - root.layer-25"_tf_keras_layer*{"name": "reshape_6", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Reshape", "config": {"name": "reshape_6", "trainable": true, "dtype": "float32", "target_shape": {"class_name": "__tuple__", "items": [64]}}, "inbound_nodes": [[["mean_hin_aggregator_3", 0, 0, {}]]], "shared_object_id": 33}2 - root.layer-26"_tf_keras_layer*{"name": "lambda", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Lambda", "config": {"name": "lambda", "trainable": true, "dtype": "float32", "function": {"class_name": "__tuple__", "items": ["4wEAAAAAAAAAAAAAAAEAAAAEAAAAUwAAAHMOAAAAdABqAXwAZAFkAo0CUwApA07p/////6kB2gRh\neGlzKQLaAUvaDGwyX25vcm1hbGl6ZakB2gF4qQByCAAAAPpQL29wdC9jb25kYS9lbnZzL3JhcGlk\ncy9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RlbGxhcmdyYXBoL2xheWVyL2hpbnNhZ2Uu\ncHnaCDxsYW1iZGE+ZgEAAPMAAAAA\n", null, null]}, "function_type": "lambda", "module": "stellargraph.layer.hinsage", "output_shape": null, "output_shape_type": "raw", "output_shape_module": null, "arguments": {}}, "inbound_nodes": [[["reshape_6", 0, 0, {}]]], "shared_object_id": 34}2 \ No newline at end of file diff --git a/models/fraud-detection-models/hinsage-model.pt/saved_model.pb b/models/fraud-detection-models/hinsage-model.pt/saved_model.pb deleted file mode 100644 index 29124702c0..0000000000 Binary files a/models/fraud-detection-models/hinsage-model.pt/saved_model.pb and /dev/null differ diff --git a/models/fraud-detection-models/hinsage-model.pt/variables/variables.data-00000-of-00001 b/models/fraud-detection-models/hinsage-model.pt/variables/variables.data-00000-of-00001 deleted file mode 100644 index f0945015bd..0000000000 Binary files a/models/fraud-detection-models/hinsage-model.pt/variables/variables.data-00000-of-00001 and /dev/null differ diff --git a/models/fraud-detection-models/hinsage-model.pt/variables/variables.index b/models/fraud-detection-models/hinsage-model.pt/variables/variables.index deleted file mode 100644 index daaaf6b9b5..0000000000 Binary files a/models/fraud-detection-models/hinsage-model.pt/variables/variables.index and /dev/null differ diff --git a/models/fraud-detection-models/hyperparams.pkl b/models/fraud-detection-models/hyperparams.pkl new file mode 100644 index 0000000000..7c87766fb0 Binary files /dev/null and b/models/fraud-detection-models/hyperparams.pkl differ diff --git a/models/fraud-detection-models/model.pt b/models/fraud-detection-models/model.pt new file mode 100644 index 0000000000..29ca394789 Binary files /dev/null and b/models/fraud-detection-models/model.pt differ diff --git a/models/fraud-detection-models/variables/variables.data-00000-of-00001 b/models/fraud-detection-models/variables/variables.data-00000-of-00001 deleted file mode 100644 index f0945015bd..0000000000 Binary files a/models/fraud-detection-models/variables/variables.data-00000-of-00001 and /dev/null differ diff --git a/models/fraud-detection-models/variables/variables.index b/models/fraud-detection-models/variables/variables.index deleted file mode 100644 index daaaf6b9b5..0000000000 Binary files a/models/fraud-detection-models/variables/variables.index and /dev/null differ diff --git a/models/fraud-detection-models/xgb.pt b/models/fraud-detection-models/xgb.pt new file mode 100644 index 0000000000..24e5212729 Binary files /dev/null and b/models/fraud-detection-models/xgb.pt differ diff --git a/models/fraud-detection-models/xgb.pth b/models/fraud-detection-models/xgb.pth deleted file mode 100644 index 4b86a49a34..0000000000 Binary files a/models/fraud-detection-models/xgb.pth and /dev/null differ diff --git a/models/training-tuning-scripts/fraud-detection-models/HinSAGE.py b/models/training-tuning-scripts/fraud-detection-models/HinSAGE.py deleted file mode 100644 index c77a0b3feb..0000000000 --- a/models/training-tuning-scripts/fraud-detection-models/HinSAGE.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pandas as pd -from stellargraph.layer import HinSAGE -from stellargraph.mapper import HinSAGENodeGenerator -from tensorflow.keras import Model -from tensorflow.keras import layers -from tensorflow.keras import optimizers -from tensorflow.keras.losses import binary_crossentropy - - -class HinSAGE_Representation_Learner: - """ - This class initializes a graphsage framework - - Parameters - ---------- - embedding_size : int - The desired size of the resulting embeddings - num_samples : list - The length of the list defines the depth of random walks, the values of the list - define the number of nodes to sample per neighborhood. - embedding_for_node_type: str - String identifying the node type for which we want graphsage to generate embeddings. - - """ - - def __init__(self, embedding_size, num_samples, embedding_for_node_type): - - self.embedding_size = embedding_size - self.num_samples = num_samples - self.embedding_for_node_type = embedding_for_node_type - - def train_hinsage(self, S, node_identifiers, label, batch_size, epochs): - """ - - This function trains a HinSAGE model, implemented in Tensorflow. - It returns the trained HinSAGE model and a pandas dataframe - containing the embeddings generated for the train nodes. - - Parameters - ---------- - S : StellarGraph Object - The graph on which HinSAGE trains its aggregator functions. - node_identifiers : list - Defines the nodes that HinSAGE uses to train its aggregation functions. - label: Pandas dataframe - Defines the label of the nodes used for training, with the index representing the nodes. - batch_size: int - batch size to train the neural network in which HinSAGE is implemented. - epochs: int - Number of epochs for the neural network. - - """ - # The mapper feeds data from sampled subgraph to GraphSAGE model - train_node_identifiers = node_identifiers[:round(0.8 * len(node_identifiers))] - train_labels = label.loc[train_node_identifiers] - - validation_node_identifiers = node_identifiers[round(0.8 * len(node_identifiers)):] - validation_labels = label.loc[validation_node_identifiers] - generator = HinSAGENodeGenerator(S, batch_size, self.num_samples, head_node_type=self.embedding_for_node_type) - train_gen = generator.flow(train_node_identifiers, train_labels, shuffle=True) - test_gen = generator.flow(validation_node_identifiers, validation_labels) - - # HinSAGE model - model = HinSAGE(layer_sizes=[self.embedding_size] * len(self.num_samples), generator=generator, dropout=0) - x_inp, x_out = model.build() - - # Final estimator layer - prediction = layers.Dense(units=1, activation="sigmoid", dtype='float32')(x_out) - - # Create Keras model for training - model = Model(inputs=x_inp, outputs=prediction) - model.compile( - optimizer=optimizers.Adam(lr=1e-3), - loss=binary_crossentropy, - ) - - # Train Model - model.fit(train_gen, epochs=epochs, verbose=1, validation_data=test_gen, shuffle=False) - - trained_model = Model(inputs=x_inp, outputs=x_out) - train_gen_not_shuffled = generator.flow(node_identifiers, label, shuffle=False) - embeddings_train = trained_model.predict(train_gen_not_shuffled) - - train_emb = pd.DataFrame(embeddings_train, index=node_identifiers) - - return trained_model, train_emb - - def inductive_step_hinsage(self, S, trained_model, inductive_node_identifiers, batch_size): - """ - This function generates embeddings for unseen nodes using a trained hinsage model. - It returns the embeddings for these unseen nodes. - - Parameters - ---------- - S : StellarGraph Object - The graph on which HinSAGE is deployed. - trained_model : Neural Network - The trained hinsage model, containing the trained and optimized aggregation functions per depth. - inductive_node_identifiers : list - Defines the nodes that HinSAGE needs to generate embeddings for - batch_size: int - batch size for the neural network in which HinSAGE is implemented. - - """ - # The mapper feeds data from sampled subgraph to HinSAGE model - generator = HinSAGENodeGenerator(S, batch_size, self.num_samples, head_node_type=self.embedding_for_node_type) - test_gen_not_shuffled = generator.flow(inductive_node_identifiers, shuffle=False) - - inductive_emb = trained_model.predict(test_gen_not_shuffled, verbose=1) - inductive_emb = pd.DataFrame(inductive_emb, index=inductive_node_identifiers) - - return inductive_emb diff --git a/models/training-tuning-scripts/fraud-detection-models/README.md b/models/training-tuning-scripts/fraud-detection-models/README.md index ac1b56d8ee..824d473bcf 100644 --- a/models/training-tuning-scripts/fraud-detection-models/README.md +++ b/models/training-tuning-scripts/fraud-detection-models/README.md @@ -17,37 +17,32 @@ limitations under the License. -## Instruction how to train new GNN models. +## Instruction how to train new GNN models. ### Setup training environment -Install packages for training GNN model. +Install packages for training GNN model. ``` -pip install -r requirements.txt +mamba env update -n ${CONDA_DEFAULT_ENV} -f requirements.yml ``` ### Options for training and tuning models. ``` python training.py --help -optional arguments: - -h, --help show this help message and exit - --training-data TRAINING_DATA - CSV with fraud_label - --validation-data VALIDATION_DATA - CSV with fraud_label - --epochs EPOCHS Number of epochs - --node_type NODE_TYPE - Target node type - --output-xgb OUTPUT_XGB - output file to save xgboost model - --output-hinsage OUTPUT_HINSAGE - output file to save GraphHinSage model - --save_model SAVE_MODEL - Save models to given filenames - --embedding_size EMBEDDING_SIZE - output file to save new model +Usage: training.py [OPTIONS] + +Options: + --training-data TEXT Path to training data + --validation-data TEXT Path to validation data + --model-dir TEXT path to model directory + --target-node TEXT Target node + --epochs INTEGER Number of epochs + --batch_size INTEGER Batch size + --output-file TEXT Path to csv inference result + --model-type TEXT Model type either RGCN/HinSAGE + --help Show this message and exit ``` @@ -59,10 +54,10 @@ export DATASET=../../dataset python training.py --training-data $DATASET/training-data/fraud-detection-training-data.csv \ --validation-data $DATASET\validation-datafraud-detection-validation-data.csv \ - --epoch 10 \ - --output-xgb model/xgb.pt \ - --output-hinsage model/hinsage.pt \ - --save_model True + --epochs 20 \ + --model_dir models\ + --model-type HinSAGE ``` +This results is a trained models of HeteroRGCN/HinSAGE (model.pt) and Gradient boosting tree (xgb.pt), hyperparmeters at the `model` directory. -This results in a trained models of GraphSAGE (hinsage.pt) and Gradient boosting tree (xgb.pt) at the `model` directory. +Note the `model.py` used for both training & inference script is a symbolink for the `../../../examples/gnn_fraud_detection_pipeline/stages/model.py`. diff --git a/models/training-tuning-scripts/fraud-detection-models/evaluation.py b/models/training-tuning-scripts/fraud-detection-models/evaluation.py deleted file mode 100644 index 9001583258..0000000000 --- a/models/training-tuning-scripts/fraud-detection-models/evaluation.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from matplotlib import pyplot -from sklearn.metrics import average_precision_score -from sklearn.metrics import f1_score -from sklearn.metrics import precision_recall_curve -from sklearn.metrics import precision_recall_fscore_support -from sklearn.metrics import roc_auc_score -from sklearn.metrics import roc_curve - - -class Evaluation: - """ - This class initializes the evaluation of a classification model. - - Parameters - ---------- - probabilities : iterable - The predicted probabilities per class for the classification model. - labels : iterable - The labels corresponding with the predicted probabilities. - name : str - The name of the used configuration - """ - - def __init__(self, probabilities, labels, name): - - self.probabilities = probabilities - self.labels = labels - self.name = name - - def pr_curve(self): - """ - This function plots the precision recall curve for the used classification model and a majority classifier. - - """ - probs = self.probabilities[:, 1] - precision, recall, _ = precision_recall_curve(self.labels, probs) - pyplot.plot(recall, precision, label=self.name) - # axis labels - pyplot.xlabel('Recall') - pyplot.ylabel('Precision') - # show the legend - pyplot.legend() - - print('Average precision-recall score for ', - self.name, - ' configuration XGBoost: {0:0.10f}'.format(average_precision_score(self.labels, probs))) - - def roc_curve(self, model_name='XGBoost'): - """ - This function plots the precision recall curve for the used classification model and a majority classifier. - - """ - probs = self.probabilities[:, 1] - fpr, tpr, _ = roc_curve(self.labels, probs) - auc = round(roc_auc_score(self.labels, probs), 3) - pyplot.plot(fpr, tpr, label=self.name + str(auc)) - # axis labels - pyplot.xlabel('FPR') - pyplot.ylabel('TPR') - # show the legend - pyplot.legend() - - print('ROC score for ', self.name, ' configuration : {0:0.10f}'.format(auc)) - return auc - - def f1_ap_rec(self): - - probs = self.probabilities[:, 1] >= 0.5 - prec, rec, f1, num = precision_recall_fscore_support(self.labels, probs, average=None) - - print("Precision:%.3f \nRecall:%.3f \nF1 Score:%.3f" % (prec[1], rec[1], f1[1])) - micro_f1 = f1_score(self.labels, probs, average='micro') - print("Micro-Average F1 Score:", micro_f1) - # return micro_f1 diff --git a/models/training-tuning-scripts/fraud-detection-models/gnn-fraud-detection-training.ipynb b/models/training-tuning-scripts/fraud-detection-models/gnn-fraud-detection-training.ipynb new file mode 100644 index 0000000000..d12f4c6bd3 --- /dev/null +++ b/models/training-tuning-scripts/fraud-detection-models/gnn-fraud-detection-training.ipynb @@ -0,0 +1,1019 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Fraud detection pipeline using Graph Neural Network" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Content\n", + "1. Introduction\n", + "1. Load transaction data\n", + "2. Graph construction\n", + "3. GraphSage training\n", + "5. Classifiction and Prediction\n", + "6. Evaluation\n", + "7. Conclusion" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Introduction\n", + "This workflow shows an application of a graph neural network for fraud detection in a credit card transaction graph. We use a transaction dataset that includes three types of nodes, `transaction`, `client`, and `merchant` nodes. We use `GraphSAGE` along `XGBoost` to identify frauds in transactions. Since the graph is heterogeneous we employ HinSAGE a heterogeneous implementation of GraphSAGE.\n", + "\n", + "First, GraphSAGE is trained separately to produce embedding of transaction nodes, then the embedding is fed to `XGBoost` classifier to identify fraud and nonfraud transactions. During the inference stage, an embedding for a new transaction is computed from the trained GraphSAGE model and then feed to XGBoost model to get the anomaly scores." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Loading the Credit Card Transaction Data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pylab as plt\n", + "import os\n", + "import dgl\n", + "import numpy as np\n", + "import pandas as pd\n", + "import torch\n", + "import torch.nn as nn\n", + "from model import HeteroRGCN\n", + "from model import HinSAGE\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.metrics import auc\n", + "from sklearn.metrics import average_precision_score\n", + "from sklearn.metrics import precision_recall_curve\n", + "from sklearn.metrics import roc_auc_score\n", + "from sklearn.metrics import roc_curve\n", + "from torchmetrics.functional import accuracy\n", + "from tqdm import trange\n", + "from xgboost import XGBClassifier\n", + "from training import (get_metrics, evaluate, init_loaders, build_fsi_graph,\n", + " map_node_id, prepare_data, save_model, train)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(1001)\n", + "torch.manual_seed(1001)\n", + "\n", + "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "device(type='cuda', index=0)" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#device " + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Load traing and test dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Replace training-data.csv and validation-data.csv with training & validation csv in dataset file.\n", + "TRAINING_DATA ='../../datasets/training-data/fraud-detection-training-data.csv'\n", + "VALIDATION_DATA = '../../datasets/validation-data/fraud-detection-validation-data.csv'\n", + "train_data = pd.read_csv(TRAINING_DATA)\n", + "inductive_data = pd.read_csv(VALIDATION_DATA)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Since the number of samples of training data is small we augment data using benign transaction examples from the original training samples. This increases the number of benign example and reduce the proportion of fraudulent transactions. This is similar to practical situation where frauds are few in proportion." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# Increase number of samples.\n", + "def augement_data(train_data=train_data, n=20):\n", + " max_id = inductive_data.index.max()\n", + " non_fraud = train_data[train_data['fraud_label'] == 0]\n", + " \n", + " non_fraud = non_fraud.drop(['index'], axis=1)\n", + " df_fraud = pd.concat([non_fraud for i in range(n)])\n", + " df_fraud.index = np.arange(1076, 1076 + df_fraud.shape[0])\n", + " df_fraud['index'] = df_fraud.index\n", + " \n", + " return pd.concat((train_data, df_fraud))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "train_data = augement_data(train_data, n=20)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `train_data` variable stores the data that will be used to construct graphs on which the representation learners can train. \n", + "The `inductive_data` will be used to test the inductive performance of our representation learning algorithms." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The distribution of fraud for the train data is:\n", + " 0 11865\n", + "1 188\n", + "Name: fraud_label, dtype: int64\n", + "The distribution of fraud for the inductive data is:\n", + " 0 244\n", + "1 21\n", + "Name: fraud_label, dtype: int64\n" + ] + } + ], + "source": [ + "print('The distribution of fraud for the train data is:\\n', train_data['fraud_label'].value_counts())\n", + "print('The distribution of fraud for the inductive data is:\\n', inductive_data['fraud_label'].value_counts())" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# split train, test and create nodes index\n", + "def prepare_data(df_train, df_test):\n", + " \n", + " train_idx_ = df_train.shape[0]\n", + " df = pd.concat([df_train, df_test], axis=0)\n", + " df['tran_id'] = df['index']\n", + "\n", + " meta_cols = ['tran_id', 'client_node', 'merchant_node']\n", + " for col in meta_cols:\n", + " map_node_id(df, col)\n", + "\n", + " train_idx = df['tran_id'][:train_idx_]\n", + " test_idx = df['tran_id'][train_idx_:]\n", + "\n", + " df['index'] = df['tran_id']\n", + " df.index = df['index']\n", + "\n", + " return (df.iloc[train_idx, :], df.iloc[test_idx, :], train_idx, test_idx, df['fraud_label'].values, df)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "train_data, test_data, train_idx, inductive_idx, labels, df = prepare_data(train_data, inductive_data)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Construct transasction graph network" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, nodes, edges, and features are passed to the `build_fsi_graph` method. Note that client and merchant node data are featurless, instead node embedding is used as a feature for these nodes. Therefore all the relevant transaction data resides at the transaction node." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "meta_cols = [\"client_node\", \"merchant_node\", \"fraud_label\", \"index\", \"tran_id\"]\n", + "\n", + "# Build graph\n", + "whole_graph, feature_tensors = build_fsi_graph(df, meta_cols)\n", + "train_graph, _ = build_fsi_graph(train_data, meta_cols)\n", + "whole_graph = whole_graph.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# Dataset to tensors\n", + "feature_tensors = feature_tensors.to(device)\n", + "train_idx = torch.from_numpy(train_idx.values).to(device)\n", + "inductive_idx = torch.from_numpy(inductive_idx.values).to(device)\n", + "labels = torch.LongTensor(labels).to(device)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Graph(num_nodes={'client': 623, 'merchant': 388, 'transaction': 12053},\n", + " num_edges={('client', 'buy', 'transaction'): 12053, ('merchant', 'sell', 'transaction'): 12053, ('transaction', 'bought', 'client'): 12053, ('transaction', 'issued', 'merchant'): 12053},\n", + " metagraph=[('client', 'transaction', 'buy'), ('transaction', 'client', 'bought'), ('transaction', 'merchant', 'issued'), ('merchant', 'transaction', 'sell')])\n" + ] + } + ], + "source": [ + "# Show structure of training graph.\n", + "print(train_graph)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4. Train Heterogeneous GraphSAGE" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "HinSAGE, a heterogeneous graph implementation of the GraphSAGE framework is trained with user specified hyperparameters. The model train several GraphSAGE models on the type of relationship between different types of nodes." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "# Hyperparameters\n", + "target_node = \"transaction\"\n", + "epochs = 20\n", + "in_size, hidden_size, out_size, n_layers,\\\n", + " embedding_size = 111, 64, 2, 2, 1\n", + "batch_size = 100\n", + "hyperparameters = {\"in_size\": in_size, \"hidden_size\": hidden_size,\n", + " \"out_size\": out_size, \"n_layers\": n_layers,\n", + " \"embedding_size\": embedding_size,\n", + " \"target_node\": target_node,\n", + " \"epoch\": epochs}\n", + "\n", + "\n", + "scale_pos_weight = train_data['fraud_label'].sum() / train_data.shape[0]\n", + "scale_pos_weight = torch.tensor(\n", + " [scale_pos_weight, 1-scale_pos_weight]).to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "# Dataloaders\n", + "train_loader, val_loader, test_loader = init_loaders(train_graph.to(\n", + " device), train_idx, test_idx=inductive_idx,\n", + " val_idx=inductive_idx, g_test=whole_graph, batch_size=batch_size)\n", + "\n", + "\n", + "# Set model variables\n", + "model = HinSAGE(train_graph, in_size, hidden_size, out_size, n_layers, embedding_size).to(device)\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)\n", + "loss_func = nn.CrossEntropyLoss(weight=scale_pos_weight.float())" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/20 [00:00#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}
XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
+       "              colsample_bylevel=None, colsample_bynode=None,\n",
+       "              colsample_bytree=None, early_stopping_rounds=None,\n",
+       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
+       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
+       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
+       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
+       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
+       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
+       "              n_estimators=100, n_jobs=None, num_parallel_tree=None,\n",
+       "              predictor=None, random_state=None, ...)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "XGBClassifier(base_score=None, booster=None, callbacks=None,\n", + " colsample_bylevel=None, colsample_bynode=None,\n", + " colsample_bytree=None, early_stopping_rounds=None,\n", + " enable_categorical=False, eval_metric=None, feature_types=None,\n", + " gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n", + " interaction_constraints=None, learning_rate=None, max_bin=None,\n", + " max_cat_threshold=None, max_cat_to_onehot=None,\n", + " max_delta_step=None, max_depth=None, max_leaves=None,\n", + " min_child_weight=None, missing=nan, monotone_constraints=None,\n", + " n_estimators=100, n_jobs=None, num_parallel_tree=None,\n", + " predictor=None, random_state=None, ...)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Train XGBoost classifier on embedding vector\n", + "classifier = XGBClassifier(n_estimators=100)\n", + "classifier.fit(train_embedding.cpu().numpy(), labels[train_seeds].cpu().numpy())\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If requested, the original transaction features are added to the generated embeddings. If these features are added, a baseline consisting of only these features (without embeddings) is included to analyze the net impact of embeddings on the predictive performance." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "xgb_pred = classifier.predict_proba(test_embedding.cpu().numpy())\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6. Evaluation" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Given the highly imbalanced nature of the dataset, we can evaluate the results based on AUC, accuracy ...etc." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Final Test Accuracy: 0.9245283018867925 auc 0.9055425448868072\n" + ] + } + ], + "source": [ + "acc, f_1, precision, recall, roc_auc, pr_auc, average_precision, _, _ = get_metrics(\n", + " xgb_pred, labels[inductive_idx].cpu().numpy(), name='HinSAGE_XGB')\n", + "print(f\"Final Test Accuracy: {acc} auc {roc_auc}\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The result shows, using GNN embedded features with XGB achieves with a better performance when tested over embedded features. " + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.2 Save models" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The graphsage and xgboost models can be saved into their respective save format using `save_model` method. For infernce, graphsage load as pytorch model, and the XGBoost load using `cuml` *Forest Inference Library (FIL)*." + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [], + "source": [ + "model_dir= \"modelpath/\"\n", + "\n", + "save_model(train_graph, model, hyperparameters, classifier, model_dir)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "graph.pkl hyperparams.pkl model.pt xgb.pt\n" + ] + } + ], + "source": [ + "!ls modelpath" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## For inference we can load from file as follows. \n", + "from training import load_model\n", + "# do inference on loaded model, as follows\n", + "# hinsage_model, hyperparam, g = load_model(model_dir, device)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 7. Conclusion" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this workflow, we show a hybrid approach how to use Graph Neural network along XGBoost for a fraud detection on credit card transaction network. For further, optimized inference pipeline refer to `Morpheus` inference pipeline of fraud detection." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Reference\n", + "1. Van Belle, Rafaël, et al. \"Inductive Graph Representation Learning for fraud detection.\" Expert Systems with Applications (2022): 116463.\n", + "2.https://stellargraph.readthedocs.io/en/stable/hinsage.html?highlight=hinsage\n", + "3.https://github.com/rapidsai/clx/blob/branch-0.20/examples/forest_inference/xgboost_training.ipynb\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/models/training-tuning-scripts/fraud-detection-models/graph-sage-fraud-detection.ipynb b/models/training-tuning-scripts/fraud-detection-models/graph-sage-fraud-detection.ipynb deleted file mode 100644 index d11a9d2ca9..0000000000 --- a/models/training-tuning-scripts/fraud-detection-models/graph-sage-fraud-detection.ipynb +++ /dev/null @@ -1,946 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Fraud detection pipeline using Graph Neural Network" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Content\n", - "1. Introduction\n", - "1. Load transaction data\n", - "2. Graph construction\n", - "3. GraphSage training\n", - "5. Classifiction and Prediction\n", - "6. Evaluation\n", - "7. Conclusion" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1. Introduction\n", - "This workflow shows an application of a graph neural network for fraud detection in a credit card transaction graph. We use a transaction dataset that includes three types of nodes, `transaction`, `client`, and `merchant` nodes. We use `GraphSAGE` along `XGBoost` to identify frauds in transactions. Since the graph is heterogeneous we employ HinSAGE a heterogeneous implementation of GraphSAGE.\n", - "\n", - "First, GraphSAGE is trained separately to produce embedding of transaction nodes, then the embedding is fed to `XGBoost` classifier to identify fraud and nonfraud transactions. During the inference stage, an embedding for a new transaction is computed from the trained GraphSAGE model and then feed to XGBoost model to get the anomaly scores." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2. Loading the Credit Card Transaction Data" - ] - }, - { - "cell_type": "code", - "execution_count": 86, - "metadata": {}, - "outputs": [], - "source": [ - "#!pip install stellargraph" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "import pandas as pd\n", - "import numpy as np\n", - "import matplotlib.pylab as plt\n", - "from evaluation import Evaluation\n", - "from HinSAGE import HinSAGE_Representation_Learner\n", - "from stellargraph import StellarGraph\n", - "import networkx as nx\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "# set seed \n", - "np.random.seed(1001)\n", - "tf.random.set_seed(1001)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### Load traing and test dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [], - "source": [ - "# Replace training-data.csv and validation-data.csv with training & validation csv in dataset file.\n", - "TRAINING_DATA ='../../datasets/training-data/fraud-detection-training-data.csv'\n", - "VALIDATION_DATA = '../../datasets/validation-data/fraud-detection-validation-data.csv'\n", - "train_data = pd.read_csv(TRAINING_DATA)\n", - "inductive_data = pd.read_csv(VALIDATION_DATA)" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
index10001001client_nodemerchant_node10041005100610071008...110411051106110711081109111011111112fraud_label
00487.50158744918181004991189...0000000001
111191.971629139195110053211923...0000000000
22231.79072066100382100530111557...0000000001
3312.3507167996509100809111123...0000100001
4458.3707174991760100484111124...0000000000
\n", - "

5 rows × 115 columns

\n", - "
" - ], - "text/plain": [ - " index 1000 1001 client_node merchant_node 1004 1005 1006 1007 \\\n", - "0 0 487.50 1 58744 91818 100499 1 1 8 \n", - "1 1 1191.97 1 62913 91951 100532 1 1 9 \n", - "2 2 231.79 0 72066 100382 100530 1 1 15 \n", - "3 3 12.35 0 71679 96509 100809 1 1 11 \n", - "4 4 58.37 0 71749 91760 100484 1 1 11 \n", - "\n", - " 1008 ... 1104 1105 1106 1107 1108 1109 1110 1111 1112 \\\n", - "0 9 ... 0 0 0 0 0 0 0 0 0 \n", - "1 23 ... 0 0 0 0 0 0 0 0 0 \n", - "2 57 ... 0 0 0 0 0 0 0 0 0 \n", - "3 23 ... 0 0 0 0 1 0 0 0 0 \n", - "4 24 ... 0 0 0 0 0 0 0 0 0 \n", - "\n", - " fraud_label \n", - "0 1 \n", - "1 0 \n", - "2 1 \n", - "3 1 \n", - "4 0 \n", - "\n", - "[5 rows x 115 columns]" - ] - }, - "execution_count": 68, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "train_data.head(5)" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [], - "source": [ - "# Set index to transaction id \n", - "train_data.index = train_data['index']\n", - "inductive_data.index = inductive_data['index']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Since the number of samples of training data is small we augment data using benign transaction examples from the original training samples. This increase the number of benign example and reduce the proportion of fraud label transactions. This in similar to practical situation where fraud labels are few in proportion." - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [], - "source": [ - "# Increase number of samples.\n", - "def augement_data(train_data=train_data, n=20):\n", - " max_id = inductive_data.index.max()\n", - " non_fraud = train_data[train_data['fraud_label'] == 0]\n", - " \n", - " non_fraud = non_fraud.drop(['index'], axis=1)\n", - " df_fraud = pd.concat([non_fraud for i in range(n)])\n", - " df_fraud.index = np.arange(1076, 1076 + df_fraud.shape[0])\n", - " df_fraud['index'] = df_fraud.index\n", - " \n", - " return pd.concat((train_data, df_fraud))" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [], - "source": [ - "train_data = augement_data(train_data, n=20)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `train_data` variable stores the data that will be used to construct graphs on which the representation learners can train. \n", - "The `inductive_data` will be used to test the inductive performance of our representation learning algorithms." - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The distribution of fraud for the train data is:\n", - " 0 11865\n", - "1 188\n", - "Name: fraud_label, dtype: int64\n", - "The distribution of fraud for the inductive data is:\n", - " 0 244\n", - "1 21\n", - "Name: fraud_label, dtype: int64\n" - ] - } - ], - "source": [ - "print('The distribution of fraud for the train data is:\\n', train_data['fraud_label'].value_counts())\n", - "print('The distribution of fraud for the inductive data is:\\n', inductive_data['fraud_label'].value_counts())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3. Construct transasction graph network" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here, nodes, edges, and features are passed to the `graph_construction` method. Note that client and merchant node data hold a dummy attribute with value 1. This is because we want all the relevant transaction data to reside at the transaction nodes and StellarGraph's current HinSAGE implementation requires all nodes to have features." - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [], - "source": [ - "def graph_construction(nodes, edges, node_features):\n", - " g_nx = nx.Graph()\n", - " # add nodes\n", - " for key, values in nodes.items():\n", - " g_nx.add_nodes_from(values, ntype=key)\n", - " # add edges\n", - " for edge in edges:\n", - " g_nx.add_edges_from(edge)\n", - "\n", - " return StellarGraph(g_nx, node_type_name=\"ntype\", node_features=node_features)" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [], - "source": [ - "def build_graph_features(dataset):\n", - " # Build graph features\n", - " transaction_node_data = dataset.drop([\"client_node\",\"merchant_node\",\"fraud_label\", \"index\"], axis=1)\n", - " #transaction_node_data = (transaction_node_data - transaction_node_data.mean(0)) / (0.0001 + transaction_node_data.std(0))\n", - " client_node_data = pd.DataFrame([1]*len(dataset.client_node.unique())).set_index(dataset.client_node.unique())\n", - " merchant_node_data = pd.DataFrame([1]*len(dataset.merchant_node.unique())).set_index(dataset.merchant_node.unique())\n", - "\n", - " nodes = {\"client\":dataset.client_node, \"merchant\":dataset.merchant_node, \"transaction\":dataset.index}\n", - " edges = [zip(dataset.client_node, dataset.index),zip(dataset.merchant_node, dataset.index)]\n", - " features = {\"transaction\": transaction_node_data, 'client': client_node_data, 'merchant': merchant_node_data}\n", - " graph = graph_construction(nodes, edges, features)\n", - " return graph" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/envs/rapids/lib/python3.7/site-packages/ipykernel_launcher.py:10: DeprecationWarning: Constructing a StellarGraph directly from a NetworkX graph has been replaced by the `StellarGraph.from_networkx` function\n", - " # Remove the CWD from sys.path while we load stuff.\n" - ] - } - ], - "source": [ - "G = build_graph_features(train_data) # Build graph using all dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "StellarGraph: Undirected multigraph\n", - " Nodes: 13064, Edges: 24106\n", - "\n", - " Node types:\n", - " transaction: [12053]\n", - " Features: float32 vector, length 111\n", - " Edge types: transaction-default->client, transaction-default->merchant\n", - " client: [623]\n", - " Features: float32 vector, length 1\n", - " Edge types: client-default->transaction\n", - " merchant: [388]\n", - " Features: float32 vector, length 1\n", - " Edge types: merchant-default->transaction\n", - "\n", - " Edge types:\n", - " merchant-default->transaction: [12053]\n", - " Weights: all 1 (default)\n", - " Features: none\n", - " client-default->transaction: [12053]\n", - " Weights: all 1 (default)\n", - " Features: none\n" - ] - } - ], - "source": [ - "print(G.info())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4. Train Heterogeneous GraphSAGE" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "HinSAGE, a heterogeneous graph implementation of the GraphSAGE framework is trained with user specified hyperparameters. The model train several GraphSAGE models on the type of relationship between different types of nodes." - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/envs/rapids/lib/python3.7/site-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n", - " super(Adam, self).__init__(name, **kwargs)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 1/10\n", - "10/10 [==============================] - 12s 1s/step - loss: 0.6418 - val_loss: 0.5367\n", - "Epoch 2/10\n", - "10/10 [==============================] - 10s 990ms/step - loss: 0.4832 - val_loss: 0.4006\n", - "Epoch 3/10\n", - "10/10 [==============================] - 10s 975ms/step - loss: 0.3759 - val_loss: 0.3133\n", - "Epoch 4/10\n", - "10/10 [==============================] - 10s 967ms/step - loss: 0.3092 - val_loss: 0.2587\n", - "Epoch 5/10\n", - "10/10 [==============================] - 10s 1s/step - loss: 0.2674 - val_loss: 0.2234\n", - "Epoch 6/10\n", - "10/10 [==============================] - 10s 971ms/step - loss: 0.2401 - val_loss: 0.1987\n", - "Epoch 7/10\n", - "10/10 [==============================] - 10s 971ms/step - loss: 0.2207 - val_loss: 0.1802\n", - "Epoch 8/10\n", - "10/10 [==============================] - 10s 979ms/step - loss: 0.2060 - val_loss: 0.1657\n", - "Epoch 9/10\n", - "10/10 [==============================] - 10s 1s/step - loss: 0.1944 - val_loss: 0.1537\n", - "Epoch 10/10\n", - "10/10 [==============================] - 10s 980ms/step - loss: 0.1848 - val_loss: 0.1436\n" - ] - } - ], - "source": [ - "# Hyper parameters to GraphSAGE\n", - "embedding_size = 64\n", - "num_samples = [2,32]\n", - "embedding_node_type = \"transaction\"\n", - "hinsage = HinSAGE_Representation_Learner(embedding_size, num_samples, embedding_node_type)\n", - "trained_hinsage_model, train_embedding = hinsage.train_hinsage(G, list(train_data.index), train_data['fraud_label'], batch_size=1000, epochs=10)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.2 Inductive Step GraphSAGE" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this part, we want to compute the inductive embedding of a new transaction. To extract the embedding of the new transactions, we need to keep indices of the original graph nodes along with the new transaction nodes. We need to concatenate the test data frame to the train data frame to create a new graph that includes all nodes." - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [], - "source": [ - "pd.options.mode.chained_assignment = None\n", - "inductive_graph_data = pd.concat((train_data, inductive_data))\n", - "inductive_graph_data = inductive_graph_data.set_index(inductive_graph_data['index'])#.drop(\"index\",axis = 1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For the inductive step, we need to add the new, unseen transactions to the graph. Because the current StellarGraph implementation does not support adding nodes and edges to an existing stellargraph object, we create a new graph that contains all the nodes from the train graph in addition to the new nodes." - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "StellarGraph: Undirected multigraph\n", - " Nodes: 13661, Edges: 24636\n", - "\n", - " Node types:\n", - " transaction: [12318]\n", - " Features: float32 vector, length 111\n", - " Edge types: transaction-default->client, transaction-default->merchant\n", - " client: [861]\n", - " Features: float32 vector, length 1\n", - " Edge types: client-default->transaction\n", - " merchant: [482]\n", - " Features: float32 vector, length 1\n", - " Edge types: merchant-default->transaction\n", - "\n", - " Edge types:\n", - " merchant-default->transaction: [12318]\n", - " Weights: all 1 (default)\n", - " Features: none\n", - " client-default->transaction: [12318]\n", - " Weights: all 1 (default)\n", - " Features: none\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/envs/rapids/lib/python3.7/site-packages/ipykernel_launcher.py:10: DeprecationWarning: Constructing a StellarGraph directly from a NetworkX graph has been replaced by the `StellarGraph.from_networkx` function\n", - " # Remove the CWD from sys.path while we load stuff.\n" - ] - } - ], - "source": [ - "G_test = build_graph_features(inductive_graph_data)\n", - "print(G_test.info())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The inductive step applies the previously learned (and optimized) aggregation functions, part of the `trained_hinsage_model`. We also pass the new graph G_test and the node identifiers (inductive_data.index) to the inductive step. " - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "53/53 [==============================] - 1s 8ms/step\n" - ] - } - ], - "source": [ - "inductive_embedding = hinsage.inductive_step_hinsage(G_test, trained_hinsage_model, inductive_data.index, batch_size=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5. Classification: predictions based on inductive embeddings" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now a selected classifier (XGBoost) can be trained using the training node embedding and test on the test node embedding." - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [], - "source": [ - "from xgboost import XGBClassifier\n", - "\n", - "classifier = XGBClassifier(n_estimators=100)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If requested, the original transaction features are added to the generated embeddings. If these features are added, a baseline consisting of only these features (without embeddings) is included to analyze the net impact of embeddings on the predictive performance." - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/envs/rapids/lib/python3.7/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n", - " warnings.warn(label_encoder_deprecation_msg, UserWarning)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[18:56:09] WARNING: /rapids/xgboost/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n", - "[18:56:09] WARNING: /rapids/xgboost/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n" - ] - } - ], - "source": [ - "col_drop = [\"fraud_label\"]\n", - "train_labels = train_data['fraud_label']\n", - "\n", - "baseline_train = train_data.drop(col_drop, axis=1)\n", - "baseline_inductive = inductive_data.drop(col_drop, axis=1)\n", - "\n", - "classifier.fit(baseline_train, train_labels)\n", - "baseline_predictions = classifier.predict_proba(baseline_inductive)\n", - " \n", - "classifier.fit(train_embedding, train_labels)\n", - "predictions = classifier.predict_proba(inductive_embedding)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 6. Evaluation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Given the highly imbalanced nature of our dataset, we evaluate the results based on precision-recall and ROC curves." - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average precision-recall score for GraphSAGE+features configuration XGBoost: 0.8021196111\n", - "Average precision-recall score for Baseline configuration XGBoost: 0.4552021575\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 84, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAAFzCAYAAAAuSjCuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA4lklEQVR4nO3deXhU5aHH8e+bkJCQfQMCCQRkERSIEBRELSqKUCh1qbbutl5csBVbrWLV21utta221lpcrnrprbduuFPUVhGXgsoiyiarLGFNQvYQyPLeP85kSGICM5CZk5P8Ps8zT2Y5mfzmPIFf3vdsxlqLiIiIeE+E2wFERETk6KjERUREPEolLiIi4lEqcREREY9SiYuIiHiUSlxERMSjurgdIFjp6ek2JyfH7RgiIh1OfX09ABERGt+1N8uWLSu01mY0f95zJZ6Tk8PSpUvdjiEiIhI2xpitLT2vP7dERASA2bNnM3v2bLdjSBBU4iIiAsCLL77Iiy++6HYMCYJKXERExKNU4iIiIh6lEhcREfEolbiIiIhHee4QMxERCY2FCxe6HUGCpJG4iIiIR6nERUQEgAcffJAHH3zQ7RgShJCVuDHmGWPMXmPMqlZeN8aYR4wxG40xXxpjRoYqi4iIHNm8efOYN2+e2zEkCKEcic8BzjvM65OAgb7bdOCxEGYRERHpcEK2Y5u19kNjTM5hFpkG/K+11gKfGGOSjTGZ1tpdocrUXEnhbrau/ChcP07CoGt8CsePnuB2DBGRsHBz7/TewPZGj/N9z32jxI0x03FG6/Tp06fNAuR/9RkjPri2zd5P2oftKR+TPWCY2zFERELOzRI3LTxnW1rQWvsk8CRAXl5ei8scjT4njmNdzKtt9XbisuI17zNm8yMcrCp3O4qIJ8XGxrodQYLkZonnA9mNHmcBO8MZIDE5jcS8s8L5IyWEPt+3Eza7nULEu9566y23I0iQ3DzE7A3gSt9e6mOA0nBuDxcREfG6kI3EjTHPAeOBdGNMPvCfQBSAtfZxYD4wGdgIVAHXhCqLiIgc2b333gvA3Xff7XISCVQo907/wRFet8CMUP18EREJznvvvQeoxL1EZ2wTERHxKJW4iIiIR6nERUREPEqXIhUREQDS0tLcjiBBUomLiAgAL7/8stsRJEiaThcREfEolbiIiAAwa9YsZs2a5XYMCYKm00VEBIDFixe7HUGCpJG4iIiIR6nERUREPEolLiIi4lHaJi4iIgBkZWW5HUGCpBIXEREAnn32WbcjSJA0nS4iIuJRKnEREQFg5syZzJw50+0YEgRNp4uICAArVqxwO4IESSNxERERj1KJi4iIeJRKXERExKO0TVxERAAYNGiQ2xEkSCpxEREB4Mknn3Q7ggRJ0+kiIiIepRIXEREApk+fzvTp092OIUHQdLqIiACwfv16tyNIkDQSFxER8SiVuIiIiEdpOl3EA0qqDrJmVxlrdpaxZlcZFdW1PPKDk4iJinQ7moi4SCUu0o5Ya9m+bz9rdpX6C3vtrnJ2lOz3LxMbFcn+mjq276tiYI8EF9NKR5Obm+t2BAmSSlzEJdU1dWzYU+Ev7LW7ylm7q4zyA7UARBg4LiOeUX1TuGJsX4ZmJjIkM5FPvy7ipr9/7nJ66YgefvhhtyNIkFTiImFQWlXDyh2lrNlVytpd5azZWcbGggrq6i0AcdGRHJ+ZyHdP6s3QXokMzUxkcM8ETZeLyGGpxEXamLWW/OL9LNtazJIt+1i6pZh1e8r9r/dMjGFor0TOGdrDX9h9UrsREWFcTC0Cl19+OQDPPvusy0kkUCpxkWNUV2/5ancZS7ccKu3dZdUAJHTtwkl9U5gyPJOT+qQwtFciqXHRLicWaVl+fr7bESRIKnGRIFUdrGXF9hJ/aX++rYQK33bszKQYRvdLZXROCnl9UxncM4FIjbBFJERU4iJHUFB+gGVb97FkSzFLtxazekcptfUWY2BwjwS+e1IvRuekkpeTSu/kWLfjikgnohIXaaayvIR1i+ZRtX4hz1SdxoLiDAC6dolgRHYy132rP3k5qYzsk0JSbJTLaUWkM1OJiwC7t21g6+JXiP36Xwzev4KRpgaAkuSujJn8U/JyUjmxVxLRXXSSQ+m4xo4d63YECZJKXDql+ro6Nnz+Afs+f5Puu97nuPqv6Qnkm0w+73kh8cOncMK/rmDK8Ew44zi344qExW9+8xu3I0iQVOLSaVRVlLJu0ZvUrPkH/UsWMZgS6qxhXdcT+ST7ZjJPvoA+A4eTFeEbbb+rHdK8qL7esqNkPxsLKti0t4KNvlt+8X5+c8Ewzjy+u9sRRdqMSlw6tN3bN7J10cvEfP0vjt+/gpNMDWV0Y0PCKWwZeB4Dx53P0LQebseUo3Cwtp4tRZVs3Osr6wKnrDcXVLK/ps6/XFpcNP3S49hdVs26PeUq8cO48MILAXj55ZddTiKBUolLh7Nv4xL2fvaSM01e1zBN3pPPe1xA/PApDD55IqOiu7odUwJUcaD20IjaV9Sb9lawdV+V/4x3AL2TYxnQPZ4x/dMY0D3euWXEkxIXzf6DdQy5520XP4U3FBUVuR1BgqQSlw5n9Jf3ONPk0SfwSU4L0+TSLtXVW7YUVbLWd7W2tbvKWLe7nJ2l1f5lukQYctLjGNQjgcnDMv1l3T8jjm7R+u9MOh/91kuH0X/UOXyy8RK6ZJ3EgFPPZ2h6T7cjSSsqDtSybnfDpVXL/YXdMA0eGWEYkBHP6H6pDOwez4DuCQzoHk/ftG5EReqPMZEGKnHpMJLSejDmxifdjiGNWGvZWVrNWv9lVZ3blqIq/zJJsVEMyUzg+ydnMyTTOZf8wB7xdO2ii7+IHIlKXETaRF29Zf2eclbuKPWX9dpd5ZTur/Evk5PWjSGZiVw4MoshmYkM6ZVIr6QYjNGRAO3B2Wef7XYECZJKXESOyp6yaj7fVsLn24tZsa2ElTtKqTroTIfHRkUyuGcC3x6e6RtdJzC4ZyLxXfVfTnt29913ux1BgqR/USJyRPsP1rFyRykrthfz+bYSVmwvYZdvh7OoSMPQzEQuzssmNzuZYVlJ5KTF6cIvImGgEheRJurrLZsLK1juK+sV20pYt6fcfzhXdmoso3NSyc1OJrdPMkMzE4mJ0vbrjmDSpEkAvPXWWy4nkUCpxEU6uaKKA6zYXuIfYX+RX0J5tXNp1YSuXRiRncyN448jNzuZEdnJpMfrGPu2VFtXz8G6+nZxiNz+/fvdjiBBcv+3RkTCqqTqIJ9s3scnm4tYtKmQ9XsqAOewrsE9Epg6ohcnZSdzUp9k+qfHE6Fp8TZRU1fP1qJK1u+pYMOeCjbsLfefYS4iApb8YgIJMboqngRHJS7SwZVV17Dk630s2lTE4k1FrN1dhrUQExXB6JxUpuX2ZnROKif2TmwXo0GvO1Bbx5bCKjbsLWf9ngo27i1nw54Kvi6spLbRGeayU2MZ1D2B1LhoFm0qouJArUpcgqZ/sSIdTOWBWpZuLWbRpkI+2VTEyh2l1FuI7hLBqD4p3DJhEGOPS2NEVrIurXoMqmvq2FxQ6R9Rr99Tzoa9FWwtOnQ6WGOgb2o3BnRPYMLQHgzqEc/A7glNzjD3/GfbWLRJpzuVo6MSF/G46po6lm0tZvGmIhZvLuKL7SXU1luiIg252cncdOYAxhyXxsg+KdoB7ShYa9lTdsA5HeyuQyet2VJYScPAOjLC0DetGwO7xzP5xEwGNiprL63zKVOmuB1BgqQSF/Go/1m0hU17K/h8WwkH6+qJjDAM653Ef5zRn7H908jLSdH0eJBq6urZVFDhP3f7Gt8Ja/ZVHvQvk5USy9DMRKYMy2RgjwQG9oinX3pchzjD3K233up2BAmS/oWLeEycr5if+2wbJ/RK5OpxOf7S1jbVwJVW1fhH1Q1fN+yp4GBdPeBsfhjcI4EJQ7ozNDORIZmJHJ+ZSFKs1rG0HypxEY85fWA6c68fy8DuCSR1U6EEY8HavSzdUszaXWXsKDl0OFV6fDRDMhO5ZlwOQ3s5hd0/PY4unexiK+PHjwdg4cKFruaQwKnERTymS2QEeTmpbsfwlC6RhoSuXVi6dR/9M+IZ2TeFy8f0ZUhmAkN7JdI9IcbtiCJHJaQlbow5D/gTEAk8Za19oNnrScCzQB9flgettf8Tykwi0vlERUbw8R1nER0ZQWy097ddizQI2VyRMSYS+AswCRgK/MAYM7TZYjOANdbaEcB44CFjTHSoMolI55UUG6UClw4nlBt8TgY2Wms3W2sPAs8D05otY4EE41yHMB7YB9SGMJOIiEiHEcrp9N7A9kaP84FTmi3zKPAGsBNIAC6x1tY3fyNjzHRgOkCfPn1CElZEpLO7+OKL3Y4gQQplibd0wmXb7PFEYAVwFnAc8C9jzEfW2rIm32Ttk8CTAHl5ec3fQ0RE2sCNN97odgQJUiin0/OB7EaPs3BG3I1dA7xiHRuBr4HjQ5hJRERaUVVVRVVVldsxJAihLPElwEBjTD/fzmrfx5k6b2wbcDaAMaYHMBjYHMJMIiLSismTJzN58mS3Y0gQQjadbq2tNcbcBLyDc4jZM9ba1caY632vPw7cC8wxxqzEmX6/3VpbGKpMIiIiHUlIjxO31s4H5jd77vFG93cC54Yyg4iISEfVuc4pKCIi0oGoxEVERDxK504XEREArr76arcjSJBU4iIiAqjEvUjT6SIiAkBhYSGFhTpAyEs0EhcREQAuuugiQNcT9xKNxEVERDxKJS4iIuJRKnERERGPUomLiIh4lHZsExERAG644Qa3I0iQVOIiIgLAJZdc4nYECZKm00VEBIDt27ezfft2t2NIEDQSFxERAK644gpAx4l7iUbiIiIiHqUSFxER8SiVuIiIiEepxEVERDxKO7aJiAgAP/vZz9yOIEFSiYuICABTp051O4IESdPpIiICwLp161i3bp3bMSQIGomLiAgA1113HaDjxL1EI3ERERGPUomLiIh4lEpcRETEo1TiIiIiHqUd20REBIC77rrL7QgSJJW4iEgHsbNkP1/tLuOUfmnEdQ3+v/cJEyaEIJWEkkpcRMSD6uot63aXs2zrPpZsKWbpln3sLK0G4Nfnn8hlp/QN+j1XrFgBQG5ubhsmlVBSiYuIeMD+g3Ws2F7C0i37WLK1mM+3FlN+oBaAHoldyctJ5bLMRH7/zjqqa+qP6mfMnDkT0HHiXqISFxFphwrKDxwaZW8tZvWOUmrrLcbAoO4JfCe3F3k5KeT1TSUrJRZjDKX7a/j9OzrjWmeiEhcRaQc2F1TywboClm51psa3FFUB0LVLBCOyk5l+Rn9G56Qysk8KSd2iXE4r7YVKXESkHbjsqU8BSI2LJq9vCpee0oe8nFRO7JVEdBcdDSwtU4mLiLho/ODu/Oi0fgzukUBeTgr90uMwxrgdSzxCJS4i4qKeSTHcPWWo2zEAuP/++92OIEFSiYuICACnnnqq2xEkSNrQIiIiACxatIhFixa5HUOCoJG4SKht+wQ++C0k94WpD7udRqRVd955J6DjxL1EJS4SKvlL4f1fw6YFzuP0Qe7mEZEORyUu0tZ2fg7v/wY2vAOxqXDOr2DLv6H4a7eTiUgHoxIXaSu7V8HC38BX8yAmGc66G065DromOMUuItLGVOIix2rvV055r3kNuibC+Fkw5gaISXI7mYh0cCpxkaNVuMHZYW3lXIiOg9NvhVNvgtgUt5OJHJWHH37Y7QgSJJW4SLD2bYYPfg9fPg9dYmDcT+DUmyEuze1kIsdElyD1HpW4SKBKtsGHv4fP/w8io+CUG+C0mRDfPbw56uugsgASeob350qH9+677wIwYcIEl5NIoFTiIkdSugM+egiW/y8YA6N/BKf9FBIzw5ujphq++Dss+jMUb4VbVoc/g3Ro9913H6AS9xKVuMjhrHoF/v0I2Do46Qo441ZIygpvhv3FsORp+PRxZwTeLd3JU12qEhfp5FTiIq3pEgPFWyD3UjjjNkjpG96fX5oPi2fDsjlQUwkDzoFxN0PlXpj7w/BmEZF2SSUu0por33B2VkvtH96fu2cNLHoEVr4E1sKwi+DUn0DPE53XV70S3jwi0m6pxEVakz06fD/LWti6CP79J+dMb1FxcPJ053jz5D7hyyEinqISF3FTfR189Q+nvHcsdbZ3n3mXs/Nct1S300kn88QTT7gdQYKkEhdxQ021c5z5oj9D0UZIyYFvPwS5l0FUrNvppJMaPHiw2xEkSCpxkXDaXwJLn3H2NK/YA5m58L05MOQ7EBHpcjjp7N58800Apk6d6nISCZRKXCRcSrbDH0+AgxVw3NlwwX9DvzOcY89F2oGHHnoIUIl7iUpcJBxiU6HuIAy5wDlMrOcwtxOJSAegEhcJh3PvhTPvhLh0t5OISAcSEco3N8acZ4xZZ4zZaIy5o5VlxhtjVhhjVhtjPghlHhHXRMepwEWkzYVsJG6MiQT+ApwD5ANLjDFvWGvXNFomGZgNnGet3WaMCfOVJERERLwrlNPpJwMbrbWbAYwxzwPTgDWNlrkUeMVauw3AWrs3hHlEROQw/va3v7kdQYIUUIkbY8YBvwT6+r7HANZae7jzUfYGtjd6nA+c0myZQUCUMWYhkAD8yVr7vy38/OnAdIA+fXT2KhGRUMjOznY7ggQp0JH408AtwDKgLsDvaem4GdvCzx8FnA3EAouNMZ9Ya9c3+SZrnwSeBMjLy2v+HiIi0gZeeOEFAC655BKXk0igAi3xUmvtW0G+dz7Q+M+6LGBnC8sUWmsrgUpjzIfACGA9IiISVo899higEveSQPdOf98Y83tjzFhjzMiG2xG+Zwkw0BjTzxgTDXwfeKPZMq8DpxtjuhhjuuFMt68N6hOIiIh0UoGOxBu2Zec1es4CZ7X2DdbaWmPMTcA7QCTwjLV2tTHmet/rj1tr1xpj3ga+BOqBp6y1q4L9ECIiIp1RQCVurT3zaN7cWjsfmN/sucebPf498PujeX8REZHOLKDpdGNMkjHmD8aYpb7bQ8aYpFCHExERkdYFOp3+DLAKuNj3+Argf4ALQhFKRETCb+7cuW5HkCAFWuLHWWsvbPT4v4wxK0KQR0REXJKerlMDe02ge6fvN8ac1vDAd/KX/aGJJCIibpgzZw5z5sxxO4YEIdCR+A3AX33bwQ2wD7g6VKFERCT8Ggr86quvZm95NZ99vY/jMuIZkpnobjBpVaB7p68ARhhjEn2Py0IZSkREwqug/ABFlQcp21/D2Q8tZFNBJQCnHpfG3/9jzDG/v7WW2npLVGRIL57Z6Ry2xI0xl1trnzXG/LTZ8wBYa/8QwmwiIhIiRRUH+GTzPj7ZXMQnm4vYsLeC3XvKiYwwnJnaje/lZfPa5zuoqas/qvevratnza4ylmwpZsnX+1i6dR8Ha+v59M4JxEZHtvGn6byONBKP831NCHUQEREJnX2VB/l0cxGLfaW9fk8FAN2iIxmdk8oFI7N4+oMk4qK7MOeakwH4cH1BwCW+/2Adn28vZumWYpZs2cfyrcVUHnQutZGdGktGQgxrd5VRebBWJd6GDlvi1tonfF//KzxxRESkLRRXHuTTr4v8o+2vdpcDTmnn5aTy3ZN6M6Z/GsN6J/mnuF/oGvjVqUuqDrJkSzFLt+zjsy37WLWjlJo6izEwuEcCF47KYnROKqNzUumZFMPfFm/h7tdXh+SzdmaBXor0d8B9OHukv41zkZKZ1tpnQ5hNRESOwiPvbeDeeWsAiI2KJC8nhakjejGmfxrDs5Ja3S49f/78Fp8H2FGy3ynsr/exZMs+/0g+OjKC4VlJXHt6f07OSWVk3xSSYqPa/kNJiwL9s+tca+3PjTHn41x57HvA+4BKXESknYiLjiSvbwoxUZGM6Z/K2OPSGNY7meguge1M1q1bt28899XucsY9sIAdJc5RxQlduzCybwrTcnszOieV4VlJxERpetwtgZZ4w59Vk4HnrLX7GnZuExGR9qFLZARzbzj1qL9/9uzZANx4440AHN8zkU0FFeRmJ/Mfp/djdL9Uju+ZSGSE/v9vLwIt8TeNMV/hTKffaIzJAKpDF0tERMLtxRdfBA6V+D1Th3LP1KFuRmqiuqaOL/NLWba1mA17yvnx2QPplx535G/swAI9TvwOY8xvgTJrbZ0xphKYFtpoIiLSme0q3c/yrSUs21rMsm3FrN5RSm299b9+Ut8UlfjhXjTGnGWtXWCMuaDRc40XeSVUwUREpPOoqatn7a4yp7C3FrN8azE7S50J365dIhiRncx/nNGfUX1SyE7txsSHP3Q5cftwpJH4t4AFwNQWXrOoxEVE5CjsqzzI59uK/aX9RX4J1TXOMem9kmIY2TeFa/ukMKpvCkMyE5vsnFdQfsCt2O3OkY4T/0/f12vCE0dERDqy/3xjNWt3lbHZd1rXLhGGE3ol8oOT+zCqbwoj+6TQKznW5ZTeEehx4vcDv7PWlvgepwA/s9beFcJsIiISRgsXLgzZe6fGdQVg0cZCRvVN4aJRWYzqk8LwrGSdwe0YBLp3+iRr7Z0ND6y1xcaYyYBKXEREjmjysJ4s+cUE0uOjm+9bJccg0MvJRBpjujY8MMbEAl0Ps7yIiHjMgw8+yIMPPhiS9zbGkJHQVQXexgIt8WeB94wxPzLG/BD4F/DX0MUSEZFwmzdvHvPmzXM7hgQh0OPEf2eM+RKYABjgXmvtOyFNJiIiIocV+CVrYC1Qa6191xjTzRiTYK0tD1UwERERObyAptONMf8BzAWe8D3VG3gtRJlEREQkAIFuE58BjAPKAKy1G4DuoQolIiLhFxsbS2ysjtH2kkCn0w9Yaw827FVojOmCc8Y2ERHpIN566y23I0iQAh2Jf2CMuROINcacA7wEvBm6WCIiInIkgZb47UABsBK4DpiPTvQiItKh3Hvvvdx7771ux5AgHHE63RgTAXxprT0R+O/QRxIRETe89957ANx9990uJ5FAHXEkbq2tB74wxvQJQx4REREJUKA7tmUCq40xnwGVDU9aa78TklQiIiJyRIGW+H+FNIWIiIgE7bAlboyJAa4HBuDs1Pa0tbY2HMFERCS80tLS3I4gQTrSSPyvQA3wETAJGArcHOpQIiISfi+//LLbESRIRyrxodbaYQDGmKeBz0IfSURERAJxpL3TaxruaBpdRKRjmzVrFrNmzXI7hgThSCPxEcaYMt99g3PGtjLffWutTQxpOhERCZvFixe7HUGCdNgSt9ZGhiuIiIhIOFlr2VpUxRf5JQzJTGRQjwS3IwUtmOuJi4iIeFZx5UFW5JewYlsJK7aX8EV+CSVVzlbj8YMzmHPNyS4nDJ5KXEREOpwDtXWs2VnmlPV2p7S3FFUBYAwM7B7PuUN7kJudwpxFX1NTV+9y4qOjEhcREQCysrLcjnBUGqbFV/jK+vPtJazdWcZBXzF3T+hKbnYyF4/OJjc7mWG9k0iIifJ//yvL892KfsxU4iIiAsCzzz7rdoSgvLNqN++u2dNkWjw2KpJhWUlcPS6H3OxkcrOTyUyKwRjjctrQUImLiIinxEZHEh0Zwb83FTaZFs/NTmZQj3i6RAZ6lW3vU4mLiAgAM2fOBODhhx92NceRxHftwke3n0lc1y7Ed+3cNda5P72IiPitWLHC7QgB65EY43aEdqHzzDmIiIh0MCpxERERj1KJi4iIeJS2iYuICACDBg1yO4IESSUuIiIAPPnkk25HkCBpOl1ERMSjVOIiIgLA9OnTmT59utsxJAiaThcREQDWr1/vdgQJkkbiIiIiHqUSFxER8SiVuIiIiEeFtMSNMecZY9YZYzYaY+44zHKjjTF1xpiLQplHRERal5ubS25urtsxJAgh27HNGBMJ/AU4B8gHlhhj3rDWrmlhud8C74Qqi4iIHFl7v3qZfFMoR+InAxuttZuttQeB54FpLSz3Y+BlYG8Is4iIiHQ4oSzx3sD2Ro/zfc/5GWN6A+cDjx/ujYwx040xS40xSwsKCto8qIiIwOWXX87ll1/udgwJQihL3LTwnG32+GHgdmtt3eHeyFr7pLU2z1qbl5GR0Vb5RESkkfz8fPLz892OIUEI5cle8oHsRo+zgJ3NlskDnjfGAKQDk40xtdba10KYS0REpEMIZYkvAQYaY/oBO4DvA5c2XsBa26/hvjFmDjBPBS4iIhKYkJW4tbbWGHMTzl7nkcAz1trVxpjrfa8fdju4iIiIHF5Iz51urZ0PzG/2XIvlba29OpRZRETk8MaOHet2BAmSLoAiIiIA/OY3v3E7ggRJp10VERHxKJW4iIgAcOGFF3LhhRe6HUOCoOl0EREBoKioyO0IEiSNxEVERDxKJS4iIuJRKnERERGP0jZxEREB4Oyzz3Y7ggRJJS4iIgDcfffdbkeQIGk6XURExKNU4iIiAsCkSZOYNGmS2zEkCJpOFxERAPbv3+92BAmSRuIiIiIepRIXERHxKJW4iIiIR2mbuIiIADBlyhS3I0iQVOIiIgLArbfe6nYECZKm00VERDxKJS4iIgCMHz+e8ePHux1DgqASFxER8SiVuIiIiEepxEVERDxKJS4iIuJROsRMREQAuPjii92OIEFSiYuICAA33nij2xEkSJpOFxERAKqqqqiqqnI7hgRBI3EREQFg8uTJACxcuNDdIBIwjcRFREQ8SiUuIiLiUSpxERERj9I2cRERkaNUV2/Zvq+K9XvK2bC3gg17yvm6sJKXbziVLpGhHyerxEVEBICrr77a7QjtVl29ZZuvrDfurXBKe08FmwoqOFBb71+ud3IsA3vEU1ZdS2pcdMhzqcRFRARQiYNT1luLKv2j6g17K1jvK+uDLZT1aQPTGdA9nkE9EhjQPZ74ruGtVZW4iIgAUFhYCEB6errLScJv9c4yJv3po1bL+nSXy7o17SOFiIi47qKLLgI633HiudnJ7CqtpkdiV04fmM7A7vEMbGdl3Zr2nU5ERCTE7poylLumDHU7xlHRIWYiIiIepRIXERHxKJW4iIiIR2mbuIiIAHDDDTe4HUGCpBIXEREALrnkErcjSJA0nS4iIgBs376d7du3ux1DgqCRuIiIAHDFFVcAne84cS/TSFxERMSjVOIiIiIepRIXERHxKJW4iIiIR2nHNhERAeBnP/uZ2xEkSCpxEREBYOrUqW5HkCBpOl1ERABYt24d69atczuGBEEjcRERAeC6664DdJy4l2gkLiIi4lEqcREREY9SiYuIiHiUSlxERMSjtGObiIgAcNddd7kdQYIU0pG4MeY8Y8w6Y8xGY8wdLbx+mTHmS99tkTFmRCjziIhI6yZMmMCECRPcjiFBCFmJG2Migb8Ak4ChwA+MMUObLfY18C1r7XDgXuDJUOUREZHDW7FiBStWrHA7hgQhlNPpJwMbrbWbAYwxzwPTgDUNC1hrFzVa/hMgK4R5RETkMGbOnAnoOHEvCeV0em9ge6PH+b7nWvMj4K2WXjDGTDfGLDXGLC0oKGjDiCIiIt4VyhI3LTxnW1zQmDNxSvz2ll631j5prc2z1uZlZGS0YUQRERHvCuV0ej6Q3ehxFrCz+ULGmOHAU8Aka21RCPOIiIh0KKEciS8BBhpj+hljooHvA280XsAY0wd4BbjCWrs+hFlEREQ6nJCNxK21tcaYm4B3gEjgGWvtamPM9b7XHwfuAdKA2cYYgFprbV6oMomISOvuv/9+tyNIkEJ6shdr7XxgfrPnHm90/1rg2lBmEBGRwJx66qluR5Ag6bSrIiICwKJFi1i0aNGRF5R2Q6ddFRERAO68805Ax4l7iUbiIiIiHqUSFxER8SiVuIiIiEepxEVERDxKO7aJiAgADz/8sNsRJEgqcRERASA3N9ftCBKkDlHiNTU15OfnU11d7XYU8ZiYmBiysrKIiopyO4qI6959910AJkyY4HISCVSHKPH8/HwSEhLIycnBd/pWkSOy1lJUVER+fj79+vVzO46I6+677z5AJe4lHWLHturqatLS0lTgEhRjDGlpaZrBERHP6hAlDqjA5ajo90ZEvKzDlHh7sGfPHi699FL69+/PqFGjGDt2LK+++mqbvHdOTg6FhYUt/swpU6YwYsQIhg4dyuTJk5u8/uqrr2KM4auvvmry/Geffcb48eMZOHAgI0eO5Nvf/jYrV64E4Je//CW9e/cmNzfXfyspKQko50svvcSQIUM488wzg/6MJSUlzJ49O+jvExHprFTibcRay3e/+13OOOMMNm/ezLJly3j++efJz89vslxtbW2b/tx77rmHc845hy+++II1a9bwwAMPNHn9ueee47TTTuP555/3P7dnzx4uvvhi7r//fjZs2MDy5cuZNWsWmzZt8i9zyy23sGLFCv8tOTm5yfv+8pe/ZM6cOd/I8/TTTzN79mzef//9oD/L0ZZ4XV1d0N8jItIRqMTbyIIFC4iOjub666/3P9e3b19+/OMfM2fOHL73ve8xdepUzj33XCoqKjj77LMZOXIkw4YN4/XXXwdgy5YtHH/88Vx11VUMHz6ciy66iKqqKv/7/fnPf/Z/T8PIeteuXWRlZfmXGT58uP9+RUUF//73v3n66aeblPijjz7KVVdd1eSyg6eddhrf/e53j2kd/OpXv+Ljjz/m+uuv57bbbqOuro7bbruN0aNHM3z4cJ544gl/rpY+/x133MGmTZvIzc3ltttuY+HChUyZMsX//jfddJP/D4ecnBx+9atfcdppp/HSSy/xz3/+k7FjxzJy5Ei+973vUVFR4X/PoUOHMnz4cG699dZj+nwiHd0TTzzh/3cq3tAh9k5v7L/eXM2anWVt+p5DeyXyn1NPOOwyq1evZuTIka2+vnjxYr788ktSU1Opra3l1VdfJTExkcLCQsaMGcN3vvMdANatW8fTTz/NuHHj+OEPf8js2bP95ZOens7y5cuZPXs2Dz74IE899RQzZszgkksu4dFHH2XChAlcc8019OrVC4DXXnuN8847j0GDBpGamsry5csZOXIkq1ev5qqrrjrs5/njH//Is88+C0BKSkpAI+t77rmHBQsW8OCDD5KXl8eTTz5JUlISS5Ys4cCBA4wbN45zzz2X7OzsFj//Aw88wKpVq1ixYgVw5CspxcTE8PHHH1NYWMgFF1zAu+++S1xcHL/97W/5wx/+wE033cSrr77KV199hTEm4E0CIp3V4MGD3Y4QWtbC/mIo3w0Vu6F8j/O1Yi8MOBsGeG+v/A5X4u3FjBkz+Pjjj4mOjmbGjBmcc845pKamAs7U+5133smHH35IREQEO3bsYM+ePQBkZ2czbtw4AC6//HIeeeQRf4lfcMEFAIwaNYpXXnkFgIkTJ7J582befvtt3nrrLU466SRWrVpFRkYGzz33HDNnzgTg+9//Ps8991yLf2iccsoplJWVce655/KnP/0JcKbTm49cV65cyRVXXAHA7t27iY6O9p/h6b333iMtLa3J8v/85z/58ssvmTt3LgClpaVs2LCBrKysVj9/MC655BIAPvnkE9asWeNfbwcPHmTs2LEkJiYSExPDtddey7e//e0mo3oR+aY333wTgKlTp7qcpBXWQnWpU8LVpdB7JERGQX0dVBY2LWb/191Qscf3eA/UHWj5vfeuUYm3B0caMYfKCSecwMsvv+x//Je//IXCwkLy8vIAiIuL87/2f//3fxQUFLBs2TKioqLIycnxH+bUfG/pxo+7du0KQGRkZJNt66mpqVx66aVceumlTJkyhQ8//JDx48ezYMECVq1ahTGGuro6jDH87ne/44QTTmD58uVMmzYNgE8//ZS5c+cyb968w37GYcOG+UfJv/zlL8nJyeHqq69udXlrLX/+85+ZOHFik+fnzJnT6udvrEuXLtTX1/sfN1+mYZ1aaznnnHN47rnnvvEen332Ge+99x7PP/88jz76KAsWLDjsZxTpzB566CHAhRJvKOeKPVC+yyle/21X0+drm/1fEd8TKgvAtrBvTEwyJPSE+B7Qd6zvfk9I6OH76nvt2QudDB7U4UrcLWeddRZ33nknjz32GDfccANAk+3ZjZWWltK9e3eioqJ4//332bp1q/+1bdu2sXjxYsaOHevfKe1wFixYwJgxY+jWrRvl5eVs2rSJPn36MHfuXK688som27e+9a1v8fHHHzNjxgxOOeUUJk6c6N8u3lrWYzFx4kQee+wxzjrrLKKioli/fj29e/du9fMnJCRQXl7u//6+ffuyZs0aDhw4QHV1Ne+9916L62PMmDHMmDGDjRs3MmDAAKqqqsjPz6dXr15UVVUxefJkxowZw4ABA9r8M4pIC2oPNC3hhtver6DvqY1KuXE57//m+0QnOIWbkAlZo53CTciEuAz4+A+QlP3NQm74Gt8DomLC/9nDTCXeRowxvPbaa9xyyy387ne/IyMjw799dv/+pr+cl112GVOnTiUvL4/c3FyOP/54/2tDhgzhr3/9K9dddx0DBw70/0HQmmXLlnHTTTf5R63XXnsto0eP5rbbbuOOO+5osuyFF17I3//+dx577DFeeOEFbr/9dnbs2EH37t1JT0/nnnvu8S/beJs4ONvXc3Jyglon1157LVu2bGHkyJFYa8nIyOC1115r9fOnpaUxbtw4TjzxRCZNmsTvf/97Lr74YoYPH87AgQM56aSTWvw5GRkZzJkzhx/84AccOOBMld13330kJCQwbdo0qqursdbyxz/+Maj8HVJ9PRR/DbtXwp5VsGc1FG2EiffDwHPcTiduOVDhlGl1CdQehI8egjJf8Q6aCCOvdJarq3VGvY2L2V/Uuw99z/59rf+sDe9AVBwkZjrl23uUU7wJPZ2Cbjxa7prQ+vuMuKRNV4FXGeuxKYS8vDy7dOnSJs+tXbuWIUOGuJSo7WzZsoUpU6awatUqt6N0Kp77/Vn1Csy9Bm78FLof3/py1WVOSe9Z5dx2r3K2+9X4Zl1MBKQNgML1cOYv4Fs/D09+aVsHyqFsJ5TtcL5WFcGISyE+w/mjrarQeb58l28ZX9H6n9sJB5ydgcfPqQRg4dVxEJPkTHEDZOY6JV25F2x9059vIg6NgBMyG916Nv0am+L8/Njkw5ezG56eCF26wlVvBP+99XVQtc9Zz5WFzh85VUWQ9yOIaLsDwIwxy6y1ec2f10hcxOsaRtcNhb17FexZCSXbDi0Tkww9h8HIq6DHCdDzRMg4HiKj4VeprkWXw2jYTly2s2lJl+U3eu5QATfxr3sgqY9T0vU1TV8zEc5INzHT+SOu37ec+wm94O0HnN+JOxdAdBx89AdY+RLEpTu/P83LObGXM7UdERnYZ0rOPvb1Emr1dc4e7P5CbijnQt/9AqgsOvRa1T6ghcHwiRdCt9D/21KJtyM5OTkahUvg3v+1MzrauwYOOsfF+0fXvfOcwu45DHqc6Pxn29IpZut1opywqq/zTTvvgNL8Q+W883NnW3HDa2U7oXQH1FQ2ewPjjHoTezUq4F6Q2BuSejuvvfVzZ5t0Yi/nltDrUEknZkJcd4hs+b/+v73kO3dEtG9H3NN/6tw6g+2fwu+OczYFNJ9taBCb4vzR0i0dMgZB3DjnflwGxKU1up/u/OEcBipxEa+J7+F8/foDp6BzL3NG1j1OgIwhEN3N3XydVcPUdWm+r6R3OKPm0h2HHpfvankvaoBti52iTeoN3Yc4hzs1FHFib18hZzqHVB3O5S8f/vXDyM72wEg5FEZeAeszmpZwt7RD9+MyIDa11T9+3NT+EonI4eWMgzu2O9sVdQGX8Gg4SUhr5dwwxV13sOn3RXZ1yjcpC3JOcwo6sbfzuGH0HJPsvHfXRNdL4oUXXgAOnYOh0zjpcufmQSpxES+KSXQ7QcdyoLz1cm54XNPsMEwTeWiU3HsUDPlO03JOzHJGcYH8oRWGbaeBeOyxx4BOWOIephIXkY7pYJWvjLc7U9yF651tmvX13yzrA6XNvtm37blhanvgOU3LuWH7c6A7dImEiEq8jURGRjJs2DCstURGRvLoo482ucDIsbr66quZMmUKF110Eddeey0//elPGTp0aJu9v8gxqal29oYv3gKp/SE9wBPr1NU6RVqyremtdDuUbHV2+r3h39+ceaivc04YUtqopP033+PDHavcLc0p5ZQc6DuuaTkn9na2PXeJPsqVIRI+KvE2Ehsb6z8l6TvvvMOsWbP44IMPQvKznnrqqZC8r0irrHUOsSn+2inq5reynfgPs8kaDde+69yvq2m5pEu2O1/LdjTb0cs4BZrcx9nJaOdy+OQx53zXjUu6bCfUN7usb3SCcwhTUhZk5flGzr7HSVmHlk/sBVGxIVxZIuGjEg+BsrIyUlJSAOeym9OmTaO4uJiamhruu+8+pk2bRmVlJRdffDH5+fnU1dVx9913c8kll7Bs2TJ++tOfUlFRQXp6OnPmzCEzM7PJ+48fP95/pbD4+Hhuvvlm5s2bR2xsLK+//jo9evSgoKCA66+/nm3bnGOFH374Yf8FQkRaVFPtFOS+Voq6+eFOCZnOSLbfGZDSz7m/5Cko+Ar+Z3Kjkm58uI5xSjS5j3Mu6+Q+TW+JWYdGwF/Nh+d/AAvvh4guvu3PWZA95lAx+0u6t3NyEpFOpuOV+Ft3OKeUbEs9h8GkBw67yP79+8nNzaW6uppdu3b5L7QRExPT4mU33377bXr16sU//vEPwDmfek1NDT/+8Y95/fXXycjI4IUXXuAXv/gFzzzzTKs/t7KykjFjxvDrX/+an//85/z3f/83d911FzfffDO33HILp512Gtu2bWPixImsXbu27daJdCwf/QHev58mJ63oEusUc0NRp/Y79Di5T8uj2bKGw6isM039jZLuHfg09aDz4IZFznZsbX8Oi4YrDop3dLwSd0nj6fTFixdz5ZVXsmrVqlYvOzps2DBuvfVWbr/9dqZMmcLpp5/OqlWrWLVqFeec45zDuq6u7huj8Oaio6P9l9gcNWoU//rXvwB49913WbNmjX+5srIyysvLSUhoZ6c7FHdFRMKYGc7244aCbhhVx3cP/hC2tjw5SESEc+y7hE16errbESRIHa/EjzBiDoexY8dSWFhIQUEB8+fPb/Gym4MGDWLZsmXMnz+fWbNmce6553L++edzwgknsHjx4oB/VlRUlP9ypY0vUVpfX8/ixYuJjdW2PzmC8+53O4G0E3PmzAE47CWGpX1pu7Ozi99XX31FXV0daWlprV52c+fOnXTr1o3LL7+cW2+9leXLlzN48GAKCgr8JV5TU8Pq1auPKsO5557Lo48+6n/cMEsgItKaOXPm+ItcvKHjjcRd0rBNHMBay1//+lciIyNbvezmypUrue2224iIiCAqKorHHnuM6Oho5s6dy09+8hNKS0upra1l5syZnHBC8FOKjzzyCDNmzGD48OHU1tZyxhln8Pjjj7flRxYREZfpUqTS6en3R8Qxfvx4ABYuXOhqDvmm1i5Fqul0ERERj1KJi4iIeJS2iYuICADz5893O4IEqcOUuLXWf6iVSKC8tk+ISCh166Zr0XtNh5hOj4mJoaioSP8hS1CstRQVFRETE+N2FJF2Yfbs2cyePdvtGBKEDjESz8rKIj8/n4KCArejiMfExMSQlZXldgyRduHFF18E4MYbb3Q5iQSqQ5R4VFQU/fr1czuGiIhIWHWI6XQREZHOSCUuIiLiUSpxERERj/LcaVeNMQXA1jZ8y3SgsA3fr7PSejx2WofHTuvw2GkdHrtQrMO+1tqM5k96rsTbmjFmaUvno5XgaD0eO63DY6d1eOy0Do9dONehptNFREQ8SiUuIiLiUSpxeNLtAB2E1uOx0zo8dlqHx07r8NiFbR12+m3iIiIiXqWRuIiIiEd1mhI3xpxnjFlnjNlojLmjhdeNMeYR3+tfGmNGupGzPQtgHV7mW3dfGmMWGWNGuJGzPTvSOmy03GhjTJ0x5qJw5vOKQNajMWa8MWaFMWa1MeaDcGds7wL495xkjHnTGPOFbx1e40bO9soY84wxZq8xZlUrr4enU6y1Hf4GRAKbgP5ANPAFMLTZMpOBtwADjAE+dTt3e7oFuA5PBVJ89ydpHQa/DhsttwCYD1zkdu72dgvwdzEZWAP08T3u7nbu9nQLcB3eCfzWdz8D2AdEu529vdyAM4CRwKpWXg9Lp3SWkfjJwEZr7WZr7UHgeWBas2WmAf9rHZ8AycaYzHAHbceOuA6ttYustcW+h58AujxYU4H8HgL8GHgZ2BvOcB4SyHq8FHjFWrsNwFqrddlUIOvQAgnGGAPE45R4bXhjtl/W2g9x1klrwtIpnaXEewPbGz3O9z0X7DKdWbDr50c4f4XKIUdch8aY3sD5wONhzOU1gfwuDgJSjDELjTHLjDFXhi2dNwSyDh8FhgA7gZXAzdba+vDE6xDC0ikd4lKkATAtPNd8t/xAlunMAl4/xpgzcUr8tJAm8p5A1uHDwO3W2jpnACQtCGQ9dgFGAWcDscBiY8wn1tr1oQ7nEYGsw4nACuAs4DjgX8aYj6y1ZSHO1lGEpVM6S4nnA9mNHmfh/HUZ7DKdWUDrxxgzHHgKmGStLQpTNq8IZB3mAc/7CjwdmGyMqbXWvhaWhN4Q6L/nQmttJVBpjPkQGAGoxB2BrMNrgAess4F3ozHma+B44LPwRPS8sHRKZ5lOXwIMNMb0M8ZEA98H3mi2zBvAlb49CscApdbaXeEO2o4dcR0aY/oArwBXaMTToiOuQ2ttP2ttjrU2B5gL3KgC/4ZA/j2/DpxujOlijOkGnAKsDXPO9iyQdbgNZyYDY0wPYDCwOawpvS0sndIpRuLW2lpjzE3AOzh7ZT5jrV1tjLne9/rjOHsCTwY2AlU4f4WKT4Dr8B4gDZjtG0nWWl1IwS/AdShHEMh6tNauNca8DXwJ1ANPWWtbPBSoMwrwd/FeYI4xZiXO1PDt1lpd3czHGPMcMB5IN8bkA/8JREF4O0VnbBMREfGozjKdLiIi0uGoxEVERDxKJS4iIuJRKnERERGPUomLiIh4lEpcpJPxXR1thTFmle8qVclt/P5bjDHpvvsVbfneItKUSlyk89lvrc211p6IcwGHGW4HEpGjoxIX6dwW47sogzHmOGPM274LhnxkjDne93wPY8yrvutKf2GMOdX3/Gu+ZVcbY6a7+BlEOq1OccY2EfkmY0wkzmk1n/Y99SRwvbV2gzHmFGA2zsUvHgE+sNae7/ueeN/yP7TW7jPGxAJLjDEv63z5IuGlEhfpfGKNMSuAHGAZztWp4oFTgZcaXT2tq+/rWcCVANbaOqDU9/xPjDHn++5nAwMBlbhIGKnERTqf/dbaXGNMEjAPZ5v4HKDEWpsbyBsYY8YDE4Cx1toqY8xCICYUYUWkddomLtJJWWtLgZ8AtwL7ga+NMd8D8F15aYRv0feAG3zPRxpjEoEkoNhX4McDY8L+AUREJS7SmVlrPwe+wLkU5WXAj4wxXwCrgWm+xW4GzvRdzWoZcALwNtDFGPMlztWuPgl3dhHRVcxEREQ8SyNxERERj1KJi4iIeJRKXERExKNU4iIiIh6lEhcREfEolbiIiIhHqcRFREQ8SiUuIiLiUf8P/9Wqw1OeJuMAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "#from inductiveGRL.evaluation import Evaluation\n", - "inductive_labels = inductive_data['fraud_label'] #df.loc[inductive_emb.index]['fraud_label']\n", - "plt.figure(figsize=(8, 6))\n", - "graphsage_evaluation = Evaluation(predictions, inductive_labels, \"GraphSAGE+features\") \n", - "graphsage_evaluation.pr_curve()\n", - "\n", - "#if add_additional_data is True:\n", - "\n", - "baseline_evaluation = Evaluation(baseline_predictions, inductive_labels, \"Baseline\")\n", - "baseline_evaluation.pr_curve()\n", - "plt.axvline(x=0.8, color='k', linestyle='--')" - ] - }, - { - "cell_type": "code", - "execution_count": 85, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ROC score for GraphSAGE+features configuration : 0.9720000000\n", - "ROC score for Baseline configuration : 0.7190000000\n" - ] - }, - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 85, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAAFzCAYAAAAuSjCuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA+AklEQVR4nO3de3zOdePH8dfHbM5yvnPK5JhhM5M5lZAzc8yolEMIORQ63b/q1uFWSZJTIuquUArF6i5JzmebswiJCHPMcbPP74/N7plh2LXvdXg/H4892vW9vruu977N3vt8Tx9jrUVEREQ8TxanA4iIiMitUYmLiIh4KJW4iIiIh1KJi4iIeCiVuIiIiIdSiYuIiHiorE4HuFmFChWygYGBTscQERHJNOvWrTtqrS2cernHlXhgYCBr1651OoaIiEimMcb8ntZy7U4XERHxUCpxERERD6USFxER8VAqcREREQ+lEhcREfFQKnEREREPpRIXERHxUCpxERERD6USFxER8VAuK3FjzEfGmMPGmM3XeN4YY8YYY3YZYzYaY0JdlUVERMQbuXIkPg1oep3nmwHlkj56ARNcmEVERMTruOze6dbaxcaYwOusEgF8Yq21wEpjTD5jTFFr7UFXZZLM8/mqfcyNPuB0DJFb0vBsFHXO/ex0DPFAcfEJnC8URHjfDzPl/Zw8Jl4c+CPF4/1Jy65ijOlljFlrjFl75MiRTAknt2du9AG2HjzldAyRW1Ln3M8Exu12OoZ4mF1/naX92GgWrMu8nx0nZzEzaSyzaa1orZ0ETAIICwtLcx1xP5WK5mVm71pOxxC5eVPvAKoR1G2+00nEg2TdsYMsUS2p/fCLmfaeTo7E9wMlUzwuAfzpUBYREZGbduzYMUaPHo21lgoVKrB9+3YaNGiQae/vZIl/A3RNOks9HDip4+EiIuIplixZQkhICMOGDWPr1q0A+Pn5ZWoGV15iNh1YAVQwxuw3xvQwxvQxxvRJWiUK2A3sAj4E+roqi4iISEa5dOkSw4cPp379+gQEBLB8+XKCgoIcyeLKs9M73+B5C/Rz1fuLiIi4QqdOnfjqq694+OGHGT9+PHnz5nUsi5MntomIiHgMay3GGLp160br1q3p2rWr05FU4iIiItdz/vx5hg0bRokSJRg2bBgtWrRwOlIy3TtdRETkGnbs2EF4eDjvv/8+7nifEo3ERUREUrHWMm3aNPr370+OHDn49ttvadmypdOxrqKRuIiISCo7duygZ8+e3HvvvcTExLhlgYNG4iIiIskOHDhA8eLFqVixIosWLaJ27dqZfu33zdBIXEREfF5CQgJvv/02d999Nz/++CMA9erVc+sCB43ERUTEx/3111889thj/Pe//6V9+/aEhYU5HSndVOJuytOn8tx68BSVijp3AwRxE2unwqZZTqe4eYc2wZ1VnE4hmeDHH3/k0Ucf5eTJk0ycOJFevXphTFrzc7kn7U53U54+lWelonmJCElzZlnxJZtmJRaip7mzClTp4HQKyQQ7d+6kYMGCrFmzht69e3tUgYNG4m5NU3mKV7izCmhKT3Eje/bs4ddff6VJkyY8+eSTdO/enezZszsd65ZoJC4iIj5jxowZhISE8MQTT3Dx4kWMMR5b4KASFxERH3DmzBl69OhB586dCQoKYvHixQQEBDgd67Zpd7qIiHi1U6dOUbNmTXbs2MELL7zAK6+8gr+/v9OxMoRKXEREvFrevHlp27YtDRs2pGHDhk7HyVDanS4iIl7n2LFjdOrUiejoaADeeOMNrytwUImLiIiXWbJkCcHBwcyePZuNGzc6HcelVOIiIuIVLl26xL/+9S/q169P9uzZWbFiBV27dnU6lkupxEVExCt8+OGHvPLKK3Tp0oX169dTvXp1pyO5nE5sExERj3bixAny5ctHjx49KFasGK1bt3Y6UqbRSFxERDzS+fPneeqpp6hSpQqxsbH4+/v7VIGDRuIiIuKBtm/fTmRkJDExMQwePJjcuXM7HckRKvFMcrOzkmkWMPFIqWct02xgksGstXz00UcMGDCAnDlzMn/+fJo3b+50LMdod3omudlZyTQLmHik1LOWaTYwcYFZs2YRHh5OTEyMTxc4aCSeqTQrmfgEzVomLrB69WqKFi1KyZIlmTlzJrly5cLPz8/pWI7TSFxERNxWQkICb731FnXq1OHZZ58FEm+jqgJPpJG4iIi4pb/++ouuXbvyww8/0L59e8aNG+d0JLejEhcREbezYcMGmjVrxsmTJ5k4cSK9evXCGON0LLej3ekiIuJ2ypQpQ82aNVmzZg29e/dWgV+DSlxERNzC7t27efzxxzl//jx58+Zl7ty5VK5c2elYbk0lLiIijps+fTohISHMmTOHLVu2OB3HY6jERUTEMWfOnKFHjx506dKFKlWqEBMT4xMTl2QUlbiIiDime/fuTJ06lRdffJFffvmFUqVKOR3Jo+jsdBERyVTWWi5cuED27Nn517/+Re/evWnQoIHTsTySSlxERDJNbGwsPXr0IHv27EyfPp2KFStSsWJFp2N5LO1OFxGRTLF48WJCQkKIiooiPDzc6TheQSNxF0o5c5lmJROvt3Yq/L4UStV1Oom4mfj4eF577TVeffVVypQpw8qVKwkNDXU6llfQSNyFUs5cplnJxOtdnoJUs5ZJKkeOHOH999/nkUceYd26dSrwDKSRuItp5jLxKaXqQlg3p1OIm1iyZAl16tShaNGibNy4keLFNZDJaBqJi4hIhjp//jz9+/fnvvvuY+rUqQAqcBfRSFxERDLMtm3biIyMZOPGjQwePJhHHnnE6UheTSUuIiIZYubMmXTv3p2cOXMyf/58mjdv7nQkr6fd6SIikiGKFStG3bp1iYmJUYFnEpW4iIjcslWrVvHOO+8AUK9ePb7//nuKFSvmcCrfoRIXEZGblpCQwJtvvkndunUZO3Ysf//9N4Dm/c5kKnEREbkphw4domnTpjz33HNERESwfv16cufO7XQsn6QT20REJN0uXLhAeHg4f/31FxMnTqRXr14afTtIJS4iIjd06dIl/Pz8yJYtG2+++SZBQUFUrlzZ6Vg+T7vTRUTkunbv3k2tWrWYMWMGAJ06dVKBuwmVuIiIXNP06dMJCQlh586d5MiRw+k4kopLd6cbY5oC7wF+wGRr7YhUz98BfArclZRlpLV2qiszZYbLs5dp5jLxOmun/m+ik9QObYI7q2RuHnGZM2fO8NRTTzF16lTq1KnDZ599RqlSpZyOJam4bCRujPEDxgHNgEpAZ2NMpVSr9QO2WmuDgfrAO8aYAFdlyiwpC1wzl4lX2TQrsazTcmcVzWDmRRYsWMC0adP45z//yaJFi1TgbsqVI/F7gV3W2t0AxpgZQASwNcU6FshjEk9tzA0cA+JdmCnTaPYy8Vp3VoFu851OIS5grWXz5s1UqVKFiIgItm7dSsWKFZ2OJdfhymPixYE/Ujzen7QspbHAPcCfwCZgoLU2wYWZREQkDbGxsbRp04YaNWqwc+dOABW4B3Bliad14aBN9bgJEA0UA0KAscaYqw4iG2N6GWPWGmPWHjlyJKNzioj4tF9++YXg4GC+++47RowYQdmyZZ2OJOnkyhLfD5RM8bgEiSPulLoBX9tEu4A9wFV/+llrJ1lrw6y1YYULF3ZZYBERXzN8+HAaNGhAzpw5WblyJYMGDdLNWzyIK0t8DVDOGFM66WS1SOCbVOvsAxoCGGP+AVQAdrswk4iIpHD27FkeeeQR1q1bR2hoqNNx5Ca57MQ2a228MaY/8F8SLzH7yFq7xRjTJ+n5icCrwDRjzCYSd78/a6096qpMIiICc+bMoWDBgtSrV4833niDLFl0yxBP5dLrxK21UUBUqmUTU3z+J9DYlRlERCTR+fPneeaZZxg/fjytW7emXr16KnAPp/97IiI+YNu2bdSsWZPx48fz9NNP88UXXzgdSTKAJkAREfFymzZtIjw8nJw5czJ//nyaN2/udCTJIBqJi4h4KWsTr+oNCgri6aefJiYmRgXuZVTiIiJeaNWqVdx777388ccfZMmShVdffZVixYo5HUsymEpcRMSLJCQk8Oabb1K3bl2OHDnC0aO64Meb6Zj4Tbg8O9mNaPYy8QppzVimmcrc2qFDh3j00UdZsGABHTt2ZNKkSeTLl8/pWOJCGonfhMuzk92IZi8Tr5DWjGWaqcytDR8+nGXLljFp0iRmzpypAvcBGonfJM1OJj5FM5a5vYsXL3L06FGKFSvGiBEj6N+/P5UqpZ71WbyVRuIiIh7qt99+o27dujRv3pz4+Hjy5s2rAvcxGomLiHigzz//nD59+uDn58eUKVPImlW/zn2RRuIiIh7kzJkzdOvWjYcffpiqVasSHR1Nu3btnI4lDlGJi4h4ED8/P2JiYvi///s/Fi1aRKlSpZyOJA7S/hcRETdnrWXKlCl07NiRO+64g5UrVxIQEOB0LHEDGomLiLixo0ePEhERwRNPPMHkyZMBVOCSTCNxERE3tWjRIh5++GGOHj3K6NGjGTBggNORxM1oJC4i4oY+/vhjGjRoQK5cuVixYgUDBw7EGON0LHEzKnERETfUsGFD+vbty/r16wkNDXU6jrgplbiIiJuYPXs2kZGRJCQkUKJECcaOHUvu3LmdjiVuTCUuIuKwc+fO0bdvX9q1a8dvv/3G8ePHnY4kHkIntqVyvZnKNDuZ+ITLs5dpxrJMsXXrViIjI9m0aRPPPPMMb7zxhs4+l3RTiadyeaaytMpas5OJT0hZ4JqxzKUuXbpEu3btOHbsGFFRUTRr1szpSOJhVOJp0Exl4vM0e5lLnTx5khw5chAQEMDnn39O0aJFKVq0qNOxxAPpmLiISCZauXIlISEhvPTSSwCEhoaqwOWWqcRFRDJBQkICI0aMoG7dugC0adPG2UDiFbQ7XUTExQ4dOsSjjz7KggULeOihh/jggw/Ily+f07HEC6jERURc7PDhw6xbt44PP/yQHj166M5rkmFU4iIiLnDx4kXmzJnDQw89RNWqVfn999/JkyeP07HEy+iYuIhIBtu1axd16tShU6dOrF27FkAFLi6hEhcRyUCfffYZ1apVY9euXXz11VeEhYU5HUm8mEpcRCSDDBgwgEceeYTg4GBiYmJo166d05HEy+mYuIhIBqlTpw758uXjpZdeImtW/XoV19NPmYjILbLWMmbMGLJnz07v3r3p1KmT05HEx2h3uojILTh69CitW7dm0KBB/PTTT1hrnY4kPkglLiJykxYtWkRwcDA//PAD7733HjNnztS13+II7U4XEbkJe/bsoVGjRpQpU4Z58+ZRrVo1pyOJD1OJi4ikw5kzZ8iVKxelS5dmxowZNG3alNy5czsdS3ycdqeLiNzA7NmzCQwMZMmSJQB06NBBBS5uQSUuInIN586do2/fvrRr147AwECKFSvmdCSRK6jERUTSsHXrVmrWrMmECRMYMmQIy5Yto0yZMk7HErmCjomLiKQhKiqKv/76i++++46mTZs6HUckTRqJi4gkOXHiBCtXrgTg6aefZvPmzSpwcWsqcRERYMWKFYSEhNCmTRvOnTtHlixZKFy4sNOxRK5LJS4iPi0hIYF///vf1KtXD2MMc+bMIUeOHE7HEkkXHRMXEZ919uxZWrduzU8//cRDDz3EBx98QL58+ZyOJZJuKnER8Vk5cuSgVKlSfPjhh/To0UO3ThWPo93pIuJTLl68yHPPPcf27dsxxjBlyhR69uypAhePpJG4iPiMXbt2ERkZybp16yhUqBAVK1Z0OpLIbdFIPIXPV+1j1Z5jTscQcc7aqfD7UqdTuMSnn35KtWrV2L17N19//TVDhgxxOpLIbVOJpzA3+gAAESHFHU4i4pBNsxL/W6WDszky2Keffsqjjz5KSEgI0dHRtG3b1ulIIhlCu9NTqVm6AF1q3uV0DBHnlKoLYd2cTpEh4uLi8Pf3p0OHDpw4cYI+ffqQNat+7Yn30EhcRLyOtZb33nuP4OBgTp06Rfbs2enfv78KXLyOS0vcGNPUGLPDGLPLGPPcNdapb4yJNsZsMcb84so8IuL9jh49SuvWrRk0aBBly5YlPj7e6UgiLuOyP0uNMX7AOOBBYD+wxhjzjbV2a4p18gHjgabW2n3GmCKuyiMi3u/nn3/mkUce4ejRo4wZM4b+/fvr0jHxaq7ct3QvsMtauxvAGDMDiAC2plinC/C1tXYfgLX2sAvziIgXs9YyfPhw8uTJw/z58wkJCXE6kojLubLEiwN/pHi8H6iZap3ygL8xZhGQB3jPWvtJ6hcyxvQCegHcdZdOOhOR/9m3bx/Zs2enSJEizJgxg9y5c5MrVy6nY4lkClceE09rH5ZN9TgrUB1oATQB/s8YU/6qL7J2krU2zFobplmFROSyr7/+muDgYPr16wfAP/7xDxW4+BRXlvh+oGSKxyWAP9NY53tr7Rlr7VFgMRDswkwi4gXOnTvHk08+Sfv27SlbtiwjRoxwOpKII1xZ4muAcsaY0saYACAS+CbVOnOBesaYrMaYnCTubt/mwkwi4uF+++037r33XiZOnMjQoUNZtmwZZcqUcTqWiCNcdkzcWhtvjOkP/BfwAz6y1m4xxvRJen6itXabMeZ7YCOQAEy21m52VSYR8Xz58uUjICCA77//niZNmjgdR8RRLr3zgbU2CohKtWxiqsdvA2+7MoeIeLYTJ07wzjvv8NJLL1GwYEHWrl2rS8dE0B3bRMTNLV++nJCQEEaMGMGyZcsAVOAiSVTiIuKWLl26xBtvvMF9991HlixZWLp0KfXr13c6lohb0Y2ERbzZ2qn/m5ksPQ5tgjuruC7PTejXrx8ffPABnTp14oMPPuCOO+5wOpKI21GJi3izTbNurpjvrOL4NKTWWowxPPnkk9SoUYPu3btr97nINajERbzdnVWg23ynU9zQxYsXef755zl9+jSTJk0iODiY4GDdNkLkenRMXEQct3PnTmrXrs2oUaMICAggISHB6UgiHkEjcRFx1KeffsqTTz6Jv78/s2fPpk2bNk5HEvEYKnERcczhw4fp27cv1apV47PPPqNkyZI3/iIRSaYSF5FMt2vXLsqUKUORIkVYunQplSpVImtW/ToSuVk6Ji4imcZay+jRowkKCmLKlCkAVK1aVQUucov0L0dEMsWRI0fo1q0b8+fPp3Xr1rRt29bpSCIeTyNxEXG5X375heDgYH788Ufef/995syZQ8GCBZ2OJeLxNBIXEZe7cOEC+fLlIyoqipCQEKfjiHgNjcRFxCX27t3LJ598AkDjxo3ZuHGjClwkg6nERSTDzZo1i5CQEAYNGsTx48cBdPKaiAuoxEUkw5w7d44+ffrQsWNHKlSowNq1a8mfP7/TsUS81k3/aWyM8QMirbWfuSCPiFyPG89KFh8fT506ddiwYQPDhg3j1VdfJSAgIFPeW8RXXbPEjTF5gX5AceAb4EegPzAEiAZU4iKZzY1nJcuaNSu9evXi7rvvpnHjxpnyniK+7noj8f8Ax4EVQE9gKBAARFhro10fTUTS5Eazkh0/fpxevXrx6KOP0rp1a/r06eN0JBGfcr0Sv9taWwXAGDMZOArcZa09nSnJRMStLV++nM6dO/Pnn39Sv359p+OI+KTrndgWd/kTa+0lYI8KXEQuXbrE66+/zn333Yefnx9Lly6lX79+TscS8UnXG4kHG2NOASbpcY4Uj621Nq/L04mI2/nuu+/45z//SWRkJBMnTuSOO+5wOpKIz7pmiVtr/TIziIi4t0OHDnHnnXfSokULfvrpJx544AGMMTf+QhFxmWvuTjfGZDfGDDLGjDXG9DLG6E4NIj7owoULPP3005QrV46dO3dijKFBgwYqcBE3cL1i/pjE4+JLgOZAEDAwM0KJiHvYuXMnkZGRrF+/nv79+1OyZEmnI4lICtcr8Uopzk6fAqzOnEgi4g7+85//0LdvXwICApgzZw4RERFORxKRVK5X4inPTo/XrjMR37Jq1SpCQ0P57LPPKFGihNNxRCQN1yvxkKSz0SHxjHSdnS7i5datW4cxhtDQUN555x2yZs2Kn5/OcRVxV9e7TjzGWps36SOPtTZris9V4CJexFrLu+++S61atXj66acByJYtmwpcxM1dr8RtpqUQEcccPnyYli1b8vTTT9O8eXO++uorpyOJSDpdb3d6EWPM09d60lo7ygV5RDLOzc745QkyeFayXbt2Ua9ePY4fP87YsWPp27evLh0T8SDXK3E/IDf/u2ObiGe52Rm/PEEGz0oWGBhI8+bNGTBgAMHBwRn2uiKSOa5X4gettcMzLYmIK7jRjF/uYu/evQwePJiJEyfyj3/8gylTpjgdSURu0fWOiWsELuJlZs2aRUhICD/99BNbt251Oo6I3KbrlXjDTEshIi519uxZevfuTceOHalQoQLR0dE88MADTscSkdt0zRK31h7LzCAi4jovvPACkyZNYtiwYSxdupS7777b6UgikgE0qYmIl7LWcvr0afLmzcv//d//0bJlSxo1auR0LBHJQNfbnS4iHur48eN07NiRxo0bExcXR8GCBVXgIl5IJS7iZZYvX05ISAhz586lffv2uuuaiBdTiYt4iUuXLvH6669z3333kTVrVpYtW8bQoUPJkkX/zEW8lf51i3iJ8+fP85///IeHHnqI9evXc++99zodSURcTCe2iXi4H3/8kdq1a5MrVy6WL19O/vz5detUER+hkbiIh7pw4QKDBg2icePGvPPOOwAUKFBABS7iQzQSF/FAv/76K5GRkWzYsIGnnnqKYcOGOR1JRBygEgc+X7WPudEH2HrwFJWKppoq3RtnwvIV3jb5SZL58+fTqVMnsmXLxty5c2ndurXTkUTEIdqdDlcUeERI8SufvDwTlnieDJ7xy11UrFiRBx54gJiYGBW4iI/TSDxJpaJ5mdm7VtpPaiYscdi6dev49NNPGTVqFGXKlOHbb791OpKIuAGNxEXcWEJCAqNGjaJWrVrMmjWLgwcPOh1JRNyISlzETR0+fJiWLVvyzDPP0KJFC2JiYihWrJjTsUTEjWh3uogbstbSpEkTtm3bxtixY+nbt68uHRORq7i0xI0xTYH3AD9gsrV2xDXWqwGsBDpZa3UquPisuLg4smTJgp+fH6NHjyZfvnwEBwc7HUtE3JTLdqcbY/yAcUAzoBLQ2RhT6RrrvQn811VZRDzB3r17uf/++3njjTcAuP/++1XgInJdrjwmfi+wy1q721p7EZgBRKSx3lPAV8BhF2YRcWtffvklISEhbNmyhfLlyzsdR0Q8hCtLvDjwR4rH+5OWJTPGFAfaAhOv90LGmF7GmLXGmLVHjhzJ8KAiTjl79iy9e/fmoYceokKFCmzYsIFOnTo5HUtEPIQrSzyts3BsqsejgWettZeu90LW2knW2jBrbVjhwoUzKp+I47Zu3crUqVN59tlnWbp0KXfffbfTkUTEg7jyxLb9QMkUj0sAf6ZaJwyYkXTWbSGguTEm3lo7x4W5RBxlrWXZsmXUrVuXsLAwdu3axV133eV0LBHxQK4cia8ByhljShtjAoBI4JuUK1hrS1trA621gcAsoK8KXLzZ8ePH6dChA/Xq1WPZsmUAKnARuWUuG4lba+ONMf1JPOvcD/jIWrvFGNMn6fnrHgcX8TZLly6lS5cuHDx4kJEjR1Kr1jVu8ysikk4uvU7cWhsFRKValmZ5W2sfd2WWm3Z59jIvnQlLMtfIkSN59tlnKV26NMuXL6dGjRpORxIRL6Dbrl5LygL3wpmwJHPlz5+fyMhI1q9frwIXkQyj265ej2Yvk9swf/58Tp8+TWRkJN27d6d79+66daqIZCiNxEUy2IULFxg0aBAtW7Zk7NixWGsxxqjARSTDqcRFMtCvv/5KrVq1eO+99xgwYAALFixQeYuIy2h3ukgGOXDgAKGhoWTPnp1vvvmGVq1aOR1JRLycSlzkNl26dAk/Pz+KFy/OiBEjaNOmDSVKlHA6loj4AO1OF7kNa9eupUqVKqxduxaA/v37q8BFJNOoxEVuQUJCAqNGjaJ27dqcPn2auLg4pyOJiA/S7nSRm3T48GEef/xxvvvuO9q0acOUKVMoUKCA07FExAdpJC5ykz766CMWLlzIuHHj+Prrr1XgIuIYjcRF0iEuLo7du3dToUIFhgwZQps2bahYsaLTsUTEx2kkLnIDe/bs4b777qN+/fqcPn2arFmzqsBFxC2oxEWu44svviAkJIRt27bx3nvvkSdPHqcjiYgk0+50oOHZKOqc+xmm3vG/hZq9zKddvHiR/v378+GHHxIeHs706dMJDAx0OpaIyBU0EgfqnPuZwLjdVy7U7GU+zd/fn8OHD/P888+zePFiFbiIuCWNxJPs9b+bIM1Y5tOstUyaNInGjRtTunRpvvrqK/z8/JyOJSJyTRqJiwDHjh2jQ4cO9OnTh4kTJwKowEXE7WkkLj5v6dKldOnShUOHDjFy5EgGDx7sdCQRkXRRiYtP+/bbb2nTpg2lS5dm+fLlhIWFOR1JRCTdtDtdfJK1FoAHHniAZ555hvXr16vARcTjqMTF58ybN4/69etz9uxZcufOzVtvvUXevHmdjiUictNU4uIzLly4wMCBA2nVqhWnTp0iNjbW6UgiIrdFJS4+YceOHYSHhzNmzBgGDBjAihUrKFmypNOxRERui05sE5/w5JNP8scff/DNN9/QqlUrp+OIiGQIlbh4rVOnTpGQkEC+fPn46KOP8Pf3p3jx4k7HEhHJMNqdLl5pzZo1hIaG8sQTTwAQGBioAhcRr6MSF6+SkJDAyJEjqV27NhcvXmTgwIFORxIRcRmf3p2+6st3yL1zNiUv/sYfAWWcjiO36fDhwzz22GN8//33tG3blsmTJ1OgQAGnY4mIuIxPj8RTFvjf5do6HUduU0JCAtu2bWPChAl89dVXKnAR8Xo+PRIH+COgDEEvLHU6htyiuLg4PvroI3r27Mmdd97Jjh07yJYtm9OxREQyhU+PxMWz7dmzh/vuu48+ffrw3XffAajARcSnqMTFI33xxReEhISwbds2vvjiC1q2bOl0JBGRTKcSF4/z8ssv06lTJypVqkR0dDQdO3Z0OpKIiCN8/pi4eJ5mzZoRFxfHv/71L/z9/Z2OIyLiGJW4uD1rLRMmTGDfvn2MGDGC8PBwwsPDnY4lIuI47U4Xt3bs2DHat29Pv3792LhxI/Hx8U5HEhFxGypxcVtLliwhJCSEefPmMXLkSObNm0fWrNp5JCJymX4jils6fvw4LVq0oEiRIixfvpywsDCnI4mIuB2VuLiVY8eOkT9/fvLnz8/cuXOpXr06efPmdTqWiIhb0u50cRvffPMN5cqV45NPPgHggQceUIGLiFyHSlwcd/78eQYMGEBERASlSpWiVq1aTkcSEfEIKnFx1I4dOwgPD+f9999n0KBBrFixgvLlyzsdS0TEI+iYuDhq69atHDhwgG+//Va3ThURuUkaiUumO3XqVPKEJW3btuW3335TgYuI3AKVuGSqNWvWEBoaSvv27Tl8+DCATl4TEblFKnHJFAkJCYwcOZLatWsTFxfHjz/+SJEiRZyOJSLi0XRMXFwuISGBVq1aERUVRbt27Zg8eTL58+d3OpaIiMdTiYvLZcmShTp16tCqVSt69+6NMcbpSCIiXkElLi4RFxfHSy+9RMOGDWnUqBEvvPCC05FERLyOSlwy3J49e+jcuTOrVq3CGEOjRo2cjiQi4pVcemKbMaapMWaHMWaXMea5NJ5/2BizMeljuTEm2JV5xPVmzpxJSEgI27dv54svvuCNN95wOpKIiNdyWYkbY/yAcUAzoBLQ2RhTKdVqe4D7rbVVgVeBSa7KI663YMECIiMjCQoKIjo6mo4dOzodSUTEq7lyJH4vsMtau9taexGYAUSkXMFau9xaezzp4UqghAvziIucPXsWgIYNGzJt2jR++eUXAgMDnQ0lIuIDXFnixYE/Ujzen7TsWnoA37kwj2Qway3jxo3j7rvvZu/evRhjeOyxx/D393c6moiIT3DliW1pXUdk01zRmAdILPG613i+F9AL4K677sqofHIbjh07Rvfu3Zk7dy7NmjUjZ86cTkcSEfE5rhyJ7wdKpnhcAvgz9UrGmKrAZCDCWhub1gtZaydZa8OstWGFCxd2SVhJvyVLlhAcHExUVBSjRo1i3rx5uvuaiIgDXDkSXwOUM8aUBg4AkUCXlCsYY+4CvgYetdb+6sIskoGmTZtG9uzZWbFiBdWrV3c6joiIz3JZiVtr440x/YH/An7AR9baLcaYPknPTwReAgoC45Pu4hVvrQ1zVSa5dfv37+fMmTNUqFCBMWPGkJCQQJ48eZyOJSLi01x6sxdrbRQQlWrZxBSf9wR6ujKD3L5vvvmGbt26Ua5cOVasWEGuXLmcjiQiImgWM7mO8+fPM2DAACIiIihVqhSffPKJ7nsuIuJGdNtVSdOff/5J8+bNiYmJYdCgQYwYMYJs2bI5HUtERFLQSFzSVKhQIYoVK8a8efN49913VeAiIm5IJS7JTp06xcCBAzl+/DgBAQFERUXRokULp2OJiMg1qMQFgDVr1lCtWjXGjRvHzz//7HQcERFJB5W4j0tISODtt9+mdu3axMfHs3jxYtq1a+d0LBERSQeVuI97+eWXGTZsGBEREURHR1O7dm2nI4mISDrp7HQfFR8fT9asWenbty+lSpWiR48eunxMRMTDaCTuYy5evMiwYcNo2rQply5domjRovTs2VMFLiLigVTiPmT37t3Uq1ePt99+m7JlyxIfH+90JBERuQ3ane4jZsyYQe/evTHG8OWXX9KhQwenI4mIyG1SifuAc+fO8fzzz1O5cmU+//xzSpUq5XQkERHJACpxL7ZlyxbKli1Ljhw5WLhwISVLliRrVv0vFxHxFjom7oWstYwdO5bq1avz2muvAVC6dGkVuIiIl9FvdS9z7Ngxunfvzty5c2nevDkDBgxwOpKIiLiIRuJeZNWqVQQHBxMVFcWoUaOYN28ehQsXdjqWiIi4iEbiXiRPnjwULFiQOXPmUL16dafjiIiIi2kk7uH279/PW2+9BUClSpXYsGGDClxExEeoxD3Y3LlzCQ4OZvjw4ezZswdAd14TEfEhKnEPdP78eZ566inatGlDYGAgGzZsoHTp0k7HEhGRTKZj4h7GWkuzZs1YtGgRgwcP5t///jfZsmVzOpaIiDhAJe4hrLVA4u7yp59+miFDhtCiRQuHU4mIiJNU4h7g5MmT9O7dm5o1azJ48GBatWrldCQREXEDOibu5lavXk21atWYNWuWZh0TEZErqMTdVEJCAm+99RZ16tTh0qVLLF68mKFDhzodS0RE3IhK3E1t2LCB5557joiICKKjo6ldu7bTkURExM3omLib2b17N3fffTfVq1dn9erVVK9eXdd+i4hImjQSdxMXL15k2LBhlC9fnsWLFwMQFhamAhcRkWvSSNwN7N69m86dO7N69Wp69+5NWFiY05FERMQDqMQdNnPmTJ544gn8/PyYNWsW7du3dzqSiIh4CJW4ww4dOkSVKlX4/PPPKVWqlNNxRETEg6jEHRATE8Off/5Js2bNGDBgAP369SNrVv2vEHGFuLg49u/fz/nz552OInJD2bNnp0SJEvj7+6drfTVHJrLWMm7cOIYMGUKZMmVo3Lgxfn5+KnARF9q/fz958uQhMDBQJ4qKW7PWEhsby/79+9M9qZXOTs8ksbGxtG3blqeeeopGjRqxaNEi/Pz8nI4l4vXOnz9PwYIFVeDi9owxFCxY8Kb2GmkImAmOHDlCaGgof/31F++++y4DBw7ULxSRTKR/b+IpbvZnVSPxTFC4cGEef/xxVq5cyaBBg/QLRcQH/fXXX3Tp0iX5Zk61atVi9uzZGfLagYGBHD16NM33bNmyJcHBwVSqVInmzZtf8fzs2bMxxrB9+/Yrlq9evZr69etTrlw5QkNDadGiBZs2bQLglVdeoXjx4oSEhCR/nDhxIl05v/zyS+655x4eeOCBm/4eT5w4wfjx42/6627WhQsX6NSpE2XLlqVmzZrs3bs3zfVmzpxJ1apVCQoKYtiwYcnLBw8enLxdypcvT758+QCIjo6mVq1aBAUFUbVqVWbOnJkxga21HvVRvXp1m1E2v17Hbn69Toa9Xkr79u2zjRs3tps2bXLJ64tI+mzdutXpCDYhIcGGh4fbCRMmJC/bu3evHTNmzBXrxcXF3dLrlypVyh45cuSq5b169bKjR49OfhwTE3PF8x07drR169a1L7/8cvKyQ4cO2VKlStlly5YlL1uyZImdPXu2tdbal19+2b799tvXzfPyyy/bqVOnXrW8SZMmduHChen4jq62Z88eGxQUdNNfFx8ff1Prjxs3zvbu3dtaa+306dPtQw89dNU6R48etSVLlrSHDx+21lrbtWtXu2DBgqvWGzNmjO3WrZu11todO3bYX3/91Vpr7YEDB+ydd95pjx8/nmaGtH5mgbU2jU7USNwF5syZQ3BwMMuXL2f37t1OxxERhy1cuJCAgAD69OmTvKxUqVI89dRTTJs2jY4dO9KqVSsaN27M33//TcOGDQkNDaVKlSrMnTsXgL1791KxYkUee+wxqlatSocOHTh79mzy673//vvJX3N5ZH3w4EFKlCiRvE7VqlWTP//7779ZtmwZU6ZMYcaMGcnLx44dy2OPPXbFfA1169alTZs2t7UNhg8fztKlS+nTpw9Dhw7l0qVLDB06lBo1alC1alU++OCD5Fxpff/PPfccv/32GyEhIQwdOpRFixbRsmXL5Nfv378/06ZNAxL3TAwfPpy6devy5Zdf8sMPP1CrVi1CQ0Pp2LEjf//9d/JrVqpUiapVqzJkyBAA5s6dy2OPPQZAhw4d+Omnn0js0P/ZvXs35cuXp3DhwgA0atSIr7766qrvefr06XTu3BmA8uXLU65cOQCKFStGkSJFOHLkyG1tU9Ax8Qx1/vx5nnnmGcaPH09oaCgzZsxI/p8mIs7717db2PrnqQx9zUrF8vJyq6DrrrNlyxZCQ0Ov+fyKFSvYuHEjBQoUID4+ntmzZ5M3b16OHj1KeHg4rVu3BmDHjh1MmTKFOnXq0L17d8aPH59cPoUKFWL9+vWMHz+ekSNHMnnyZPr160enTp0YO3YsjRo1olu3bhQrVgxIHGw0bdqU8uXLU6BAAdavX09oaChbtmxJLrFreffdd/n0008ByJ8/Pz///PMNt9NLL73EwoULGTlyJGFhYUyaNIk77riDNWvWcOHCBerUqUPjxo0pWbJkmt//iBEj2Lx5M9HR0QAsWrTouu+XPXt2li5dytGjR2nXrh0LFiwgV65cvPnmm4waNYr+/fsze/Zstm/fjjEm+ZDAgQMHKFmyJABZs2bljjvuIDY2lkKFCiW/dtmyZdm+fTt79+6lRIkSzJkzh4sXL17x/r///jt79uyhQYMGV2VbvXo1Fy9epEyZMjfcbjeikXgGev/99xk/fjyDBw9m+fLlKnARSVO/fv0IDg6mRo0aADz44IMUKFAASDzE+cILL1C1alUaNWrEgQMH+OuvvwAoWbIkderUAeCRRx5h6dKlya/Zrl07AKpXr558HLdJkybs3r2bJ554gu3bt1OtWrXk0d/06dOJjIwEIDIykunTp6eZtWbNmtxzzz0MHDgwedngwYOJjo4mOjo6ucA3bdqUfCx44sSJvPTSS8mPY2Njr3rdH374gU8++YSQkBBq1qxJbGwsO3fuvO73fzM6deoEwMqVK9m6dSt16tQhJCSEjz/+mN9//528efOSPXt2evbsyddff03OnDmTt39qqc9jyp8/PxMmTKBTp07Uq1ePwMDAqy4VnjFjBh06dLjqKqSDBw/y6KOPMnXqVLJkuf0K1kj8NllrOXLkCEWKFGHgwIGEhYXd0kkbIuJ6Nxoxu0pQUNAVu1vHjRvH0aNHk+dJyJUrV/Jzn332GUeOHGHdunX4+/sTGBiYfMlR6jJJ+ThbtmwA+Pn5ER8fn7y8QIECdOnShS5dutCyZUsWL15M/fr1WbhwIZs3b8YYw6VLlzDG8NZbbxEUFMT69euJiIgAYNWqVcyaNYt58+Zd93usUqVK8ij5lVdeITAwkMcff/ya61tref/992nSpMkVy6dNm3bN7z+lrFmzkpCQkPw49TqXt6m1lgcffDDNP1JWr17NTz/9xIwZMxg7diwLFy6kRIkS/PHHH5QoUYL4+HhOnjyZ/AdWSq1ataJVq1YATJo06aqynjFjBuPGjbti2alTp2jRogWvvfYa4eHh19w2N0Mj8dtw8uRJOnfuTI0aNThx4gQBAQEqcBG5SoMGDTh//jwTJkxIXpbyeHZKJ0+epEiRIvj7+/Pzzz/z+++/Jz+3b98+VqxYASSOpOvWrXvd9124cGHy+5w+fZrffvuNu+66i1mzZtG1a1d+//139u7dyx9//EHp0qVZunQp/fr1Y9q0aSxfvvyGWW9HkyZNmDBhAnFxcQD8+uuvnDlz5prff548eTh9+nTy15cqVYqtW7dy4cIFTp48yU8//ZTm+4SHh7Ns2TJ27dqV/L38+uuv/P3335w8eZLmzZszevTo5D9AWrduzccffwzArFmzaNCgQZpXFB0+fBiA48ePM378eHr27Jn83I4dOzh+/Di1atVKXnbx4kXatm1L165d6dix461utqtoJH6LVq1aRefOndm3bx+vvvoqefLkcTqSiLgpYwxz5sxh8ODBvPXWWxQuXDj5+Oy5c+euWPfhhx+mVatWhIWFERISQsWKFZOfu+eee/j444/p3bs35cqV48knn7zu+65bt47+/fsnj1p79uxJjRo1GDp0KM8999wV67Zv357PP/+cCRMmMHPmTJ599lkOHDhAkSJFKFSoEC+99FLyuimPiUPi8fXAwMCb2iY9e/Zk7969hIaGYq2lcOHCzJkz55rff8GCBalTpw6VK1emWbNmvP322zz00ENUrVqVcuXKUa1atTTfp3DhwkybNo3OnTtz4cIFAF577TXy5MlDREQE58+fx1rLu+++C0CPHj149NFHKVu2LAUKFLjipL+QkJDksh84cCAxMTFA4vH+8uXLJ693+VBFyvL/4osvWLx4MbGxsckn4E2bNo2QkJCb2m6pmbT2/7uzsLAwu3bt2gx5rS1vJP4VG/TC0hus+T8JCQm8/fbb/POf/6R48eJ8/vnnV5zFKSLuZdu2bdxzzz1Ox7hte/fupWXLlmzevNnpKOJiaf3MGmPWWWuvmqdau9NvwY8//kibNm2Ijo5WgYuIiGO0Oz2dfvjhBypXrkyxYsWYO3cuOXPm1J3XRCTTBAYGahQuV9FI/AYuXrzIsGHDaNKkCcOHDwcSz3pUgYuIiNM0Er+O3bt3ExkZyZo1a+jTpw+jRo1yOpKIiEgylfg1LFmyhBYtWuDn58esWbNo376905FERESuoN3p11ClShVatGhBdHS0ClxERNySSjyF6OhoIiMjuXDhAvny5WP69OmUKlXK6Vgi4uH8/PwICQkhODiY0NDQK26kkhEef/xxZs2aBSRef71169Zbeh1rLQMGDKBs2bJUrVqV9evXp7levXr1km+pWqxYseTJUbZv306tWrXIli0bI0eOvOJr3nvvPSpXrkxQUBCjR4++pXxyNZfuTjfGNAXeA/yAydbaEameN0nPNwfOAo9ba9P+qXGhy7f/Gzp0KIUKFWLPnj1X3GBBROR25MiRI/kmIf/97395/vnn+eWXX1zyXpMnT77lr/3uu+/YuXMnO3fuZNWqVTz55JOsWrXqqvWWLFmS/Hn79u2Tb9FaoEABxowZw5w5c65Yf/PmzXz44YesXr2agIAAmjZtSosWLTS/RAZw2UjcGOMHjAOaAZWAzsaYSqlWawaUS/roBUwgkx0/E0dERAQDBw6kcePGxMTEqMBFxGVOnTpF/vz5gWtPu3nmzBlatGhBcHAwlStXZubMmUDiHdjuv/9+qlevTpMmTTh48OBVr1+/fn0u3xArd+7cvPjiiwQHBxMeHp48kciRI0do3749NWrUoEaNGixbtgxInIaza9euGGMIDw/nxIkTab7HZadPn2bhwoXJI/EiRYpQo0YN/P39r1hv27ZthIeHkzNnTrJmzcr999/P7Nmzb2MrymWuHInfC+yy1u4GMMbMACKAlPt5IoBPkiY8X2mMyWeMKWqtvfZPTQZ77stfWfN7NKNHj2bAgAG6dEzEm333HBzalLGveWcVaDbiuqucO3eOkJAQzp8/z8GDB1m4cCGQOF1mWtNufv/99xQrVoz58+cDifdTj4uL46mnnmLu3LkULlyYmTNn8uKLL/LRRx9d833PnDlDeHg4r7/+OsOGDePDDz/kn//8JwMHDmTw4MHUrVuXffv20aRJE7Zt23bFNJwAJUqU4MCBAxQtWjTN1589ezYNGzYkb9681/3+K1euzIsvvkhsbCw5cuQgKioqefIXuT2uLPHiwB8pHu8HaqZjneLAFSVujOlF4kidu+66K8MCns53D/063UmFti9cd65fEZHbkXJ3+ooVK+jatSubN29OnnZz8eLFZMmSJXnazSpVqjBkyBCeffZZWrZsSb169di8eTObN2/mwQcfBODSpUvXLNfLAgICaNmyJZA4RemPP/4IwIIFC644bn7q1ClOnz6drmk4U5o+ffoVE39cyz333MOzzz7Lgw8+SO7cuQkODr5q6k65Na7cimn9n0/9E5KedbDWTgImQeK9028/WqLwvh+SMZPBiYhHuMGIOTPUqlWLo0ePcuTIEaKiotKcdrN8+fKsW7eOqKgonn/+eRo3bkzbtm0JCgpKnsUsPfz9/ZNLOOUUpQkJCaxYsYIcOXJcsf7laTgv279/P8WKFUvztWNjY1m9enW6d4v36NGDHj16APDCCy9QokSJdH8fcm2uPDt9P1AyxeMSwJ+3sI6IiNfYvn07ly5domDBgtecdvPPP/8kZ86cPPLIIwwZMoT169dToUIFjhw5klzicXFxbNmy5ZYyNG7cmLFjxyY/TjkN5yeffIK1lpUrV3LHHXdcc7T/5Zdf0rJlS7Jnz56u97w8dee+ffv4+uuv6dy58y1llyu5ciS+BihnjCkNHAAigS6p1vkG6J90vLwmcDIzj4eLiGSGy8fEIfFqmI8//hg/P79rTru5adMmhg4dSpYsWfD392fChAkEBAQwa9YsBgwYwMmTJ4mPj2fQoEEEBQXddJ4xY8bQr18/qlatSnx8PPfddx8TJ06kefPmREVFUbZsWXLmzMnUqVOTv6Z58+ZMnjw5eWQ+Y8aMq6YzPXToEGFhYZw6dYosWbIwevRotm7dSt68eWnfvj2xsbH4+/szbty45JP75Pa4dCpSY0xzYDSJl5h9ZK193RjTB8BaOzHpErOxQFMSLzHrZq297jyjGTkVqYh4P2+ZilR8x81MRerSMwustVFAVKplE1N8boF+rswgIiLirXTHNhEREQ+lEhcREfFQKnER8XquPPdHJCPd7M+qSlxEvFr27NmJjY1VkYvbs9YSGxub7sv2QPOJi4iXK1GiBPv37+fIkSNORxG5oezZs9/UjXBU4iLi1fz9/SldurTTMURcQrvTRUREPJRKXERExEOpxEVERDyUS2+76grGmCPA7xn4koWAoxn4er5K2/H2aRvePm3D26dtePtcsQ1LWWsLp17ocSWe0Ywxa9O6H63cHG3H26dtePu0DW+ftuHty8xtqN3pIiIiHkolLiIi4qFU4jDJ6QBeQtvx9mkb3j5tw9unbXj7Mm0b+vwxcREREU+lkbiIiIiH8pkSN8Y0NcbsMMbsMsY8l8bzxhgzJun5jcaYUCdyurN0bMOHk7bdRmPMcmNMsBM53dmNtmGK9WoYYy4ZYzpkZj5PkZ7taIypb4yJNsZsMcb8ktkZ3V06/j3fYYz51hgTk7QNuzmR010ZYz4yxhw2xmy+xvOZ0ynWWq//APyA34C7gQAgBqiUap3mwHeAAcKBVU7ndqePdG7D2kD+pM+baRve/DZMsd5CIAro4HRud/tI589iPmArcFfS4yJO53anj3RuwxeAN5M+LwwcAwKczu4uH8B9QCiw+RrPZ0qn+MpI/F5gl7V2t7X2IjADiEi1TgTwiU20EshnjCma2UHd2A23obV2ubX2eNLDlUD6p+LxDen5OQR4CvgKOJyZ4TxIerZjF+Bra+0+AGuttuWV0rMNLZDHGGOA3CSWeHzmxnRf1trFJG6Ta8mUTvGVEi8O/JHi8f6kZTe7ji+72e3Tg8S/QuV/brgNjTHFgbbAxEzM5WnS87NYHshvjFlkjFlnjOmaaek8Q3q24VjgHuBPYBMw0FqbkDnxvEKmdIqvTEVq0liW+rT89Kzjy9K9fYwxD5BY4nVdmsjzpGcbjgaetdZeShwASRrSsx2zAtWBhkAOYIUxZqW19ldXh/MQ6dmGTYBooAFQBvjRGLPEWnvKxdm8RaZ0iq+U+H6gZIrHJUj86/Jm1/Fl6do+xpiqwGSgmbU2NpOyeYr0bMMwYEZSgRcCmhtj4q21czIloWdI77/no9baM8AZY8xiIBhQiSdKzzbsBoywiQd4dxlj9gAVgdWZE9HjZUqn+Mru9DVAOWNMaWNMABAJfJNqnW+ArklnFIYDJ621BzM7qBu74TY0xtwFfA08qhFPmm64Da21pa21gdbaQGAW0FcFfpX0/HueC9QzxmQ1xuQEagLbMjmnO0vPNtxH4p4MjDH/ACoAuzM1pWfLlE7xiZG4tTbeGNMf+C+JZ2V+ZK3dYozpk/T8RBLPBG4O7ALOkvhXqCRJ5zZ8CSgIjE8aScZbTaSQLJ3bUG4gPdvRWrvNGPM9sBFIACZba9O8FMgXpfNn8VVgmjFmE4m7hp+11mp2syTGmOlAfaCQMWY/8DLgD5nbKbpjm4iIiIfyld3pIiIiXkclLiIi4qFU4iIiIh5KJS4iIuKhVOIiIiIeSiUuIgAkzZoWneIjMGkmsJPGmA3GmG3GmJeT1k25fLsxZqTT+UV8kU9cJy4i6XLOWhuScoExJhBYYq1taYzJBUQbY+YlPX15eQ5ggzFmtrV2WeZGFvFtGomLSLok3cJ0HYn30U65/ByJ99jWhEEimUwlLiKX5UixK3126ieNMQVJnBd5S6rl+YFywOLMiSkil2l3uohcdtXu9CT1jDEbSLx96Yik23PWT1q+kcR7ao+w1h7KtKQiAqjEReTGllhrW15ruTGmPLA06Zh4dCZnE/Fp2p0uIrclaca6fwPPOp1FxNeoxEUkI0wE7jPGlHY6iIgv0SxmIiIiHkojcREREQ+lEhcREfFQKnEREREPpRIXERHxUCpxERERD6USFxER8VAqcREREQ+lEhcREfFQ/w/IaB59rKW1NgAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "plt.figure(figsize=(8, 6))\n", - "graphsage_evaluation.roc_curve()\n", - "baseline_evaluation.roc_curve()\n", - "plt.plot(np.arange(0,1.1,0.1), np.arange(0,1.1,0.1),'k--')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The result show, using GNN embedded features achieves better performance compared to the XGBoost trained on raw features. We see a significant improvement in precision and recall over the GNN on raw features." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 6.2 Save models" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can save the two trained models, a graphsage and xgboost into their respective save format. For infernce, graphsage is loaded as tensorflow model, and the XGBoost is loaded using `cuml` *Forest Inference Library (FIL)*." - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "model_dir= \"model/\"\n", - "#model = {}\n", - "graph_sage_model = os.path.join(model_dir, 'fd_graphsage.pth')\n", - "xgb_model = os.path.join(model_dir, 'xgb.pth')\n", - "trained_hinsage_model.save(output_hinsage)\n", - "classifier.save_model(output_xgboost)" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [], - "source": [ - "## For inference we can load from file as follows. \n", - "from cuml import ForestInference\n", - "\n", - "graph_sage = tf.keras.models.load_model(graph_sage_model)\n", - "fm = ForestInference.load(xgb_model, output_class=True)\n", - "\n", - "# do inference on loaded models." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 7. Conclusion" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this workflow, we show a hybrid approach how to use Graph Neural network along XGBoost for a fraud detection on credit card transaction network. For further, optimized inference pipeline refer to `Morpheus` inference pipeline of fraud detection." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Reference\n", - "1. Van Belle, Rafaël, et al. \"Inductive Graph Representation Learning for fraud detection.\" Expert Systems with Applications (2022): 116463.\n", - "2.https://stellargraph.readthedocs.io/en/stable/hinsage.html?highlight=hinsage\n", - "3.https://github.com/rapidsai/clx/blob/branch-0.20/examples/forest_inference/xgboost_training.ipynb\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/models/training-tuning-scripts/fraud-detection-models/model.py b/models/training-tuning-scripts/fraud-detection-models/model.py new file mode 120000 index 0000000000..96ac13f676 --- /dev/null +++ b/models/training-tuning-scripts/fraud-detection-models/model.py @@ -0,0 +1 @@ +../../../examples/gnn_fraud_detection_pipeline/stages/model.py \ No newline at end of file diff --git a/models/training-tuning-scripts/fraud-detection-models/requirements.txt b/models/training-tuning-scripts/fraud-detection-models/requirements.txt deleted file mode 100644 index 45d60f1018..0000000000 --- a/models/training-tuning-scripts/fraud-detection-models/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -dateparser==1.1.1 -matplotlib==3.7.1 -networkx==2.8.8 -numpy==1.23.5 -pandas==1.3.5 -scikit_learn==1.1.2 -stellargraph==1.2.1 -tensorflow==2.12.0 -xgboost==1.6.2 \ No newline at end of file diff --git a/models/training-tuning-scripts/fraud-detection-models/requirements.yml b/models/training-tuning-scripts/fraud-detection-models/requirements.yml new file mode 100644 index 0000000000..7fe973ff1d --- /dev/null +++ b/models/training-tuning-scripts/fraud-detection-models/requirements.yml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +channels: + - rapidsai + - nvidia + - conda-forge +dependencies: + - click==8.1.3 + - cuml=23.06 + - dgl==1.0.2+cu118 + - numpy==1.23.5 + - pandas==1.5.3 + - scikit_learn==1.2.2 + - torch==2.0.0+cu118 + - torchmetrics==0.11.4 + - tqdm==4.65.0 + - xgboost==1.7.1 diff --git a/models/training-tuning-scripts/fraud-detection-models/training.py b/models/training-tuning-scripts/fraud-detection-models/training.py index 67873cc39e..e8facb1655 100644 --- a/models/training-tuning-scripts/fraud-detection-models/training.py +++ b/models/training-tuning-scripts/fraud-detection-models/training.py @@ -13,217 +13,524 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -# EXample usage: -python training.py --training-data ../../datasets/training-data/fraud-detection-training-data.csv \ - --validation-data ../../datasets/validation-data/fraud-detection-validation-data.csv \ - --epoch 10 --output-xgb model/xgb.pt --output-hinsage model/hinsage.pt +python training.py --training-data ../../datasets/training-data/fraud-detection-training-data.csv\ + --validation-data ../../datasets/validation-data/fraud-detection-validation-data.csv \ + --model-dir ../../fraud-detection-models --output-file out.txt\ + --model-type HinSAGE """ -import argparse +import os +import pickle -import networkx as nx +import click +import dgl import numpy as np import pandas as pd -import tensorflow as tf -from evaluation import Evaluation -from stellargraph import StellarGraph -from stellargraph.layer import HinSAGE -from stellargraph.mapper import HinSAGENodeGenerator -from tensorflow.keras import Model -from tensorflow.keras import layers -from tensorflow.keras import optimizers -from tensorflow.keras.losses import binary_crossentropy +import torch +from model import HeteroRGCN +from model import HinSAGE +from sklearn.metrics import accuracy_score +from sklearn.metrics import auc +from sklearn.metrics import average_precision_score +from sklearn.metrics import precision_recall_curve +from sklearn.metrics import roc_auc_score +from sklearn.metrics import roc_curve +from torch import nn +from torchmetrics.functional import accuracy +from tqdm import trange from xgboost import XGBClassifier -tf.random.set_seed(1001) +import cudf as cf +from cuml import ForestInference +np.random.seed(1001) +torch.manual_seed(1001) -def graph_construction(nodes, edges, node_features): - g_nx = nx.Graph() - # add nodes - for key, values in nodes.items(): - g_nx.add_nodes_from(values, ntype=key) - # add edges - for edge in edges: - g_nx.add_edges_from(edge) +def get_metrics(pred, labels, name='HinSAGE'): + """Compute evaluation metrics. - return StellarGraph(g_nx, node_type_name="ntype", node_features=node_features) + Parameters + ---------- + pred : numpy.ndarray + Predictions made by the model. + labels : numpy.ndarray + Groundtruth labels. + name : str, optional + Model name. Defaults to 'RGCN'. + + Returns + ------- + List[List] + List of evaluation metrics including: + - f1: F1-score + - precision: Precision score + - recall: Recall score + - roc_auc: Area under the Receiver Operating Characteristic curve + - pr_auc: Area under the Precision-Recall curve + - ap: Average Precision + - confusion_matrix: Confusion matrix as a list of lists + - auc_r: AUC-ROC (Area Under the ROC curve) + """ + pred, pred_proba = pred.argmax(1), pred[:, 1] -def build_graph_features(dataset): + acc = ((pred == labels)).sum() / len(pred) - transaction_node_data = dataset.drop(["client_node", "merchant_node", "fraud_label", "index"], axis=1) - client_node_data = pd.DataFrame([1] * len(dataset.client_node.unique())).set_index(dataset.client_node.unique()) - merchant_node_data = pd.DataFrame([1] * len(dataset.merchant_node.unique())).set_index( - dataset.merchant_node.unique()) + true_pos = (np.where(pred == 1, 1, 0) + np.where(labels == 1, 1, 0) > 1).sum() + false_pos = (np.where(pred == 1, 1, 0) + np.where(labels == 0, 1, 0) > 1).sum() + false_neg = (np.where(pred == 0, 1, 0) + np.where(labels == 1, 1, 0) > 1).sum() + true_neg = (np.where(pred == 0, 1, 0) + np.where(labels == 0, 1, 0) > 1).sum() - nodes = {"client": dataset.client_node, "merchant": dataset.merchant_node, "transaction": dataset.index} - edges = [zip(dataset.client_node, dataset.index), zip(dataset.merchant_node, dataset.index)] - features = {"transaction": transaction_node_data, 'client': client_node_data, 'merchant': merchant_node_data} - graph = graph_construction(nodes, edges, features) + precision = true_pos / (true_pos + false_pos) if (true_pos + false_pos) > 0 else 0 + recall = true_pos / (true_pos + false_neg) if (true_pos + false_neg) > 0 else 0 - return graph + f_1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 + confusion_matrix = pd.DataFrame(np.array([[true_pos, false_pos], [false_neg, true_neg]]), + columns=["labels positive", "labels negative"], + index=["predicted positive", "predicted negative"]) + average_precision = average_precision_score(labels, pred_proba) -def split_train_test(df, ratio=0.7, train_anom_prop=0.1, test_anom_prop=0.1): - cutoff = round(ratio * len(df)) - train_data = df.head(cutoff) - test_data = df.tail(len(df) - cutoff) + fpr, tpr, _ = roc_curve(labels, pred_proba) + prc, rec, _ = precision_recall_curve(labels, pred_proba) + roc_auc = auc(fpr, tpr) + pr_auc = auc(rec, prc) + auc_r = (fpr, tpr, roc_auc, name) + return (acc, f_1, precision, recall, roc_auc, pr_auc, average_precision, confusion_matrix, auc_r) - train_fraud = np.random.choice(train_data[train_data.fraud_label == 1].index, - int((1 - train_anom_prop) * train_data.shape[0])) - test_fraud = np.random.choice(test_data[test_data.fraud_label == 1].index, - int((1 - test_anom_prop) * test_data.shape[0])) - train_data, test_data = train_data[~train_data.index.isin( - train_fraud)], test_data[~test_data.index.isin(test_fraud)] - return train_data, test_data, train_data.index, test_data.index +def build_fsi_graph(train_data, col_drop): + """Build a heterogeneous graph from an edgelist and node index. + Parameters + ---------- + train_data : cudf.DataFrame + Training data containing node features. + col_drop : list + List of features to drop from the node features. + Returns + ------- + tuple + A tuple containing the following elements: + DGLGraph + The built DGL graph representing the heterogeneous graph. + torch.tensor + Normalized feature tensor after dropping specified columns. + Notes + ----- + This function takes the training data, represented as a cudf DataFrame, + and constructs a heterogeneous graph (DGLGraph) from the given edgelist + and node index. + + The `col_drop` list specifies which features should be dropped from the + node features to build the normalized feature tensor. + + Example + ------- + >>> import cudf + >>> train_data = cudf.DataFrame({'node_id': [1, 2, 3], + ... 'feature1': [0.1, 0.2, 0.3], + ... 'feature2': [0.4, 0.5, 0.6]}) + >>> col_drop = ['feature2'] + >>> graph, features = build_heterograph(train_data, col_drop) + """ + feature_tensors = train_data.drop(col_drop, axis=1).values + feature_tensors = torch.from_dlpack(feature_tensors.toDlpack()) + feature_tensors = (feature_tensors - feature_tensors.mean(0, keepdim=True)) / (0.0001 + + feature_tensors.std(0, keepdim=True)) -def data_preprocessing(training_dataset): + client_tensor, merchant_tensor, transaction_tensor = torch.tensor_split( + torch.from_dlpack(train_data[col_drop].values.toDlpack()).long(), 3, dim=1) - # Load dataset - df = pd.read_csv(training_dataset) - train_data, test_data, train_data_index, test_data_index = split_train_test(df, 0.7, 1.0, 0.7) - return train_data, test_data, train_data_index, test_data_index + client_tensor, merchant_tensor, transaction_tensor = (client_tensor.view(-1), + merchant_tensor.view(-1), + transaction_tensor.view(-1)) + edge_list = { + ('client', 'buy', 'transaction'): (client_tensor, transaction_tensor), + ('transaction', 'bought', 'client'): (transaction_tensor, client_tensor), + ('transaction', 'issued', 'merchant'): (transaction_tensor, merchant_tensor), + ('merchant', 'sell', 'transaction'): (merchant_tensor, transaction_tensor) + } -def train_model(train_graph, node_identifiers, label): - # train_graph: Stellar graph structure. - # Train graphsage and GBT model. + graph = dgl.heterograph(edge_list) - # Global parameters: - batch_size = 5 - xgb_n_estimator = 100 - n_samples = [2, 32] + return graph, feature_tensors - # The mapper feeds data from sampled subgraph to GraphSAGE model - train_node_identifiers = node_identifiers[:round(0.8 * len(node_identifiers))] - train_labels = label.loc[train_node_identifiers] - validation_node_identifiers = node_identifiers[round(0.8 * len(node_identifiers)):] - validation_labels = label.loc[validation_node_identifiers] - generator = HinSAGENodeGenerator(train_graph, batch_size, n_samples, head_node_type=EMBEDDING_NODE_TYPE) - train_gen = generator.flow(train_node_identifiers, train_labels, shuffle=True) - test_gen = generator.flow(validation_node_identifiers, validation_labels) +def prepare_data(training_data, test_data): + """Process data for training/inference operation - # HinSAGE model - model = HinSAGE(layer_sizes=[embedding_size] * len(n_samples), generator=generator, dropout=0) - x_inp, x_out = model.build() + Parameters + ---------- + training_data : str + path to training data + test_data : str + path to test/validation data + + Returns + ------- + tuple + tuple of (training_data, test_data, train_index, test_index, label, combined data) + """ + df_train = cf.read_csv(training_data) + df_test = cf.read_csv(test_data) + train_size = df_train.shape[0] + cdf = cf.concat([df_train, df_test], axis=0) + labels = cdf['fraud_label'].values + cdf.drop(['fraud_label', 'index'], inplace=True, axis=1) - # Final estimator layer - prediction = layers.Dense(units=1, activation="sigmoid", dtype='float32')(x_out) + cdf.reset_index(inplace=True) + meta_cols = ['client_node', 'merchant_node'] + for col in meta_cols: + cdf[col] = cf.CategoricalIndex(cdf[col]).codes - # Create Keras model for training - model = Model(inputs=x_inp, outputs=prediction) - model.compile( - optimizer=optimizers.Adam(lr=1e-3), - loss=binary_crossentropy, - ) + return (cdf.iloc[:train_size, :], + cdf.iloc[train_size:, :], + cdf['index'][:train_size], + cdf['index'][train_size:], + labels, + cdf) - # Train Model - model.fit(train_gen, epochs=epochs, verbose=1, validation_data=test_gen, shuffle=False) - hinsage_model = Model(inputs=x_inp, outputs=x_out) - train_gen_not_shuffled = generator.flow(node_identifiers, label, shuffle=False) - embeddings_train = hinsage_model.predict(train_gen_not_shuffled) +def save_model(graph, model, hyperparameters, xgb_model, model_dir): + """ Save the trained model and associated data to the specified directory. - inductive_embedding = pd.DataFrame(embeddings_train, index=node_identifiers) + Parameters + ---------- + graph : dgl.DGLGraph + The graph object representing the data used for training the model. - xgb_model = XGBClassifier(n_estimators=xgb_n_estimator) - xgb_model.fit(inductive_embedding, label) + model : nn.Module + The trained model object to be saved. - return {"hinsage": hinsage_model, "xgb": xgb_model} + hyperparameters : dict + A dictionary containing the hyperparameters used for training the model. + xgb_model : XGBoost + The trained XGBoost model associated with the main model. -def save_model(model, output_xgboost, output_hinsage): - # model: dict of xgb & hsg model - # Save as tensorflow model file + model_dir : str + The directory path where the model and associated data will be saved. - model['hinsage'].save(output_hinsage) - model['xgb'].save_model(output_xgboost) + """ + if not os.path.exists(model_dir): + os.mkdir(model_dir) + torch.save(model.state_dict(), os.path.join(model_dir, 'model.pt')) + with open(os.path.join(model_dir, 'hyperparams.pkl'), 'wb') as f: + pickle.dump(hyperparameters, f) + with open(os.path.join(model_dir, 'graph.pkl'), 'wb') as f: + pickle.dump(graph, f) + xgb_model.save_model(os.path.join(model_dir, "xgb.pt")) -def inductive_step_hinsage(graph: StellarGraph, trained_model, inductive_node_identifiers: list, batch_size: int): - """ - This function generates embeddings for unseen nodes using a trained hinsage model. - It returns the embeddings for these unseen nodes. +def load_model(model_dir, gnn_model=HinSAGE): + """Load trained models from model directory Parameters ---------- - graph : StellarGraph Object - The graph on which HinSAGE is deployed. - trained_model: Model - The trained hinsage model, containing the trained and optimized aggregation functions per depth. - inductive_node_identifiers : list - Defines the nodes that HinSAGE needs to generate embeddings for - batch_size: int - batch size for the neural network in which HinSAGE is implemented. + model_dir : str + models directory path + gnn_model: nn.Module + GNN model type to load either HinSAGE or HeteroRGCN + + Returns + ------- + (nn.Module, dgl.DGLHeteroGraph, dict) + model, training graph, hyperparameter + """ + with open(os.path.join(model_dir, "graph.pkl"), 'rb') as f: + graph = pickle.load(f) + with open(os.path.join(model_dir, 'hyperparams.pkl'), 'rb') as f: + hyperparameters = pickle.load(f) + model = gnn_model(graph, + in_size=hyperparameters['in_size'], + hidden_size=hyperparameters['hidden_size'], + out_size=hyperparameters['out_size'], + n_layers=hyperparameters['n_layers'], + embedding_size=hyperparameters['embedding_size'], + target=hyperparameters['target_node']) + model.load_state_dict(torch.load(os.path.join(model_dir, 'model.pt'))) + xgb_model = ForestInference.load(os.path.join(model_dir, 'xgb.pt'), output_class=True) + + return model, xgb_model, graph + + +def init_loaders(g_train, train_idx, test_idx, val_idx, g_test, target_node='transaction', batch_size=100): """ + Initialize dataloader and graph sampler. For training, use neighbor sampling. - # The mapper feeds data from sampled subgraph to HinSAGE model - generator = HinSAGENodeGenerator(graph, batch_size, num_samples, head_node_type="transaction") - test_gen_not_shuffled = generator.flow(inductive_node_identifiers, shuffle=False) + Parameters + ---------- + g_train : DGLHeteroGraph + Train graph. + train_idx : list + Train feature index. + test_idx : list + Test feature index. + val_idx : list + Validation index. + g_test : DGLHeteroGraph + Test graph. + target_node : str, optional + Target node. Defaults to 'transaction'. + batch_size : int + Batch size for inference. + Returns + ------- + List[tuple] + List of data loaders consisting of (DataLoader, DataLoader, DataLoader). + + + Example + ------- + >>> train_loader, test_loader, val_loader = initialize_dataloader(g_train, train_idx, test_idx, + val_idx, g_test, target_node='authentication', batch_size=32) + """ - inductive_emb = np.concatenate([trained_model.predict(row[0], verbose=1) for row in test_gen_not_shuffled]) - inductive_emb = pd.DataFrame(inductive_emb, index=inductive_node_identifiers) + neighbor_sampler = dgl.dataloading.MultiLayerFullNeighborSampler(2) + full_sampler = dgl.dataloading.MultiLayerNeighborSampler([4, 3]) + + train_dataloader = dgl.dataloading.DataLoader(g_train, {target_node: train_idx}, + neighbor_sampler, + batch_size=batch_size, + shuffle=False, + drop_last=False, + num_workers=0, + use_uva=False) + + test_dataloader = dgl.dataloading.DataLoader(g_test, {target_node: test_idx}, + full_sampler, + batch_size=batch_size, + shuffle=False, + drop_last=False, + num_workers=0, + use_uva=False) + + val_dataloader = dgl.dataloading.DataLoader(g_test, {target_node: val_idx}, + neighbor_sampler, + batch_size=batch_size, + shuffle=False, + drop_last=False, + num_workers=0, + use_uva=False) + + return train_dataloader, val_dataloader, test_dataloader + + +def train(model, loss_func, train_dataloader, labels, optimizer, feature_tensors, target_node='transaction'): + """ + Train the specified GNN model using the given training data. - return inductive_emb + Parameters + ---------- + model : torch.nn.Module + The PyTorch model to be trained. + loss_func : callable + The loss function to compute the training loss. + train_dataloader : dgl.dataloading.DataLoader + DataLoader containing the training dataset. + labels : torch.Tensor + The ground truth labels for the training data. + Shape: (num_samples, num_classes). + optimizer : torch.optim.Optimizer + The optimizer used to update the model's parameters during training. + feature_tensors : torch.Tensor + The feature tensors corresponding to the training data. + Shape: (num_samples, num_features). + target_node : str, optional + The target node for training, indicating the node of interest. + Defaults to 'transaction'. + + Returns + ------- + List[float, float] + Training accuracy and training loss + """ + model.train() + train_loss = 0.0 + for _, (_, output_nodes, blocks) in enumerate(train_dataloader): + seed = output_nodes[target_node] + nid = blocks[0].srcnodes[target_node].data[dgl.NID] + input_features = feature_tensors[nid] -def model_eval(trained_model, graph, node_identifier, label): + logits, _ = model(blocks, input_features) + loss = loss_func(logits, labels[seed]) - inductive_emb = inductive_step_hinsage(graph, trained_model['hinsage'], node_identifier, batch_size=5) - predictions = trained_model['xgb'].predict_proba(inductive_emb) - # evaluate performance. - eval_obj = Evaluation(predictions, label, "GraphSAGE+features") - eval_obj.f1_ap_rec() - print(f"AUC -- {eval_obj.roc_curve()}") + optimizer.zero_grad() + loss.backward() + optimizer.step() + train_loss += loss.item() + train_acc = accuracy_score(logits.argmax(1).cpu(), labels[seed].cpu().long()).item() + return train_acc, train_loss -def main(): - print("Data Preprocessing...") - train_data = pd.read_csv(args.training_data) - val_data = pd.read_csv(args.validation_data) - val_data.index = val_data['index'] - # train_data, val_data, train_data_index, val_data_index = split_train_test(df, 0.7, 1.0,0.7) +@torch.no_grad() +def evaluate(model, eval_loader, feature_tensors, target_node): + """Evaluate the specified model on the given evaluation input graph - print("Graph construction") - s_graph = build_graph_features(train_data) - print("Model Training...") - model = train_model(s_graph, node_identifiers=list(train_data.index), label=train_data['fraud_label']) - # print(model) - print("Save trained model") - if args.save_model: - save_model(model, args.output_xgb, args.output_hinsage) - # Save graph info - print("Model Evaluation...") - inductive_data = pd.concat((train_data, val_data)) - s_graph = build_graph_features(inductive_data) - model_eval(model, s_graph, node_identifier=list(val_data.index), label=val_data['fraud_label']) + Parameters + ---------- + model : torch.nn.Module + The PyTorch model to be evaluated. + eval_loader : dgl.dataloading.DataLoader + DataLoader containing the evaluation dataset. + feature_tensors : torch.Tensor + The feature tensors corresponding to the evaluation data. + Shape: (num_samples, num_features). + target_node : str + The target node for evaluation, indicating the node of interest. + + Returns + ------- + (torch.Tensor, torch.Tensor, torch.Tensor) + A tuple containing numpy arrays of logits, eval seed and embeddings + """ + model.eval() + eval_logits = [] + eval_seeds = [] + embedding = [] + + for _, output_nodes, blocks in eval_loader: + + seed = output_nodes[target_node] + + nid = blocks[0].srcnodes[target_node].data[dgl.NID] + input_features = feature_tensors[nid] + logits, embedd = model.infer(blocks, input_features) + eval_logits.append(logits.cpu().detach()) + eval_seeds.append(seed) + embedding.append(embedd) + + eval_logits = torch.cat(eval_logits) + eval_seeds = torch.cat(eval_seeds) + embedding = torch.cat(embedding) + return eval_logits, eval_seeds, embedding + + +@click.command() +@click.option('--training-data', help="Path to training data ", default="data/training.csv") +@click.option('--validation-data', help="Path to validation data", default="data/validation.csv") +@click.option('--model-dir', help="path to model directory", default="debug") +@click.option('--target-node', help="Target node", default="transaction") +@click.option('--epochs', help="Number of epochs", default=20) +@click.option('--batch_size', help="Batch size", default=1024) +@click.option('--output-file', help="Path to csv inference result", default="debug/out.csv") +@click.option('--model-type', help="Model type either RGCN/Graphsage", default="RGCN") +def train_model(training_data, validation_data, model_dir, target_node, epochs, batch_size, output_file, model_type): + from timeit import default_timer as timer + start = timer() + + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + if model_type == "RGCN": + gnn_model = HeteroRGCN + else: + gnn_model = HinSAGE + + # process training data + train_data, _, train_idx, inductive_idx, labels, df = prepare_data(training_data, validation_data) + + meta_cols = ["client_node", "merchant_node", "index"] + + # Build graph + whole_graph, feature_tensors = build_fsi_graph(df, meta_cols) + train_graph, _ = build_fsi_graph(train_data, meta_cols) + + feature_tensors = feature_tensors.float() + train_idx = torch.from_dlpack(train_idx.values.toDlpack()).long() + inductive_idx = torch.from_dlpack(inductive_idx.values.toDlpack()).long() + labels = torch.from_dlpack(labels.toDlpack()).long() + + # Hyperparameters + in_size, hidden_size, out_size, n_layers, embedding_size = 111, 64, 2, 2, 1 + hyperparameters = { + "in_size": in_size, + "hidden_size": hidden_size, + "out_size": out_size, + "n_layers": n_layers, + "embedding_size": embedding_size, + "target_node": target_node, + "epoch": epochs + } + + scale_pos_weight = (labels[train_idx].sum() / train_data.shape[0]).item() + scale_pos_weight = torch.FloatTensor([scale_pos_weight, 1 - scale_pos_weight]).to(device) + + # Dataloaders + train_loader, val_loader, test_loader = init_loaders(train_graph, train_idx, test_idx=inductive_idx, + val_idx=inductive_idx, g_test=whole_graph, + batch_size=batch_size) + + # Set model variables + model = gnn_model(whole_graph, in_size, hidden_size, out_size, n_layers, embedding_size).to(device) + optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4) + loss_func = nn.CrossEntropyLoss(weight=scale_pos_weight.float()) + + for epoch in trange(epochs): + + train_acc, loss = train(model, loss_func, train_loader, labels, optimizer, feature_tensors, target_node) + print(f"Epoch {epoch}/{epochs} | Train Accuracy: {train_acc} | Train Loss: {loss}") + val_logits, val_seed, _ = evaluate(model, val_loader, feature_tensors, target_node) + val_accuracy = accuracy(val_logits.argmax(1), labels.long()[val_seed].cpu(), "binary").item() + val_auc = roc_auc_score( + labels.long()[val_seed].cpu().numpy(), + val_logits[:, 1].numpy(), + ) + print(f"Validation Accuracy: {val_accuracy} auc {val_auc}") + + # Create embeddings + _, train_seeds, train_embedding = evaluate(model, train_loader, feature_tensors, target_node) + test_logits, test_seeds, test_embedding = evaluate(model, test_loader, feature_tensors, target_node) + + # compute metrics + test_acc = accuracy(test_logits.argmax(dim=1), labels.long()[test_seeds].cpu(), "binary").item() + test_auc = roc_auc_score(labels.long()[test_seeds].cpu().numpy(), test_logits[:, 1].numpy()) + + metrics_result = pd.DataFrame() + print(f"Final Test Accuracy: {test_acc} auc {test_auc}") + acc, f_1, precision, recall, roc_auc, pr_auc, average_precision, _, _ = get_metrics( + test_logits.numpy(), labels[test_seeds].cpu().numpy()) + metrics_result = [{ + 'model': 'RGCN', + 'acc': acc, + 'f1': f_1, + 'precision': precision, + 'recall': recall, + 'roc_auc': roc_auc, + 'pr_auc': pr_auc, + 'ap': average_precision + }] + + # Train XGBoost classifier on embedding vector + classifier = XGBClassifier(n_estimators=100) + classifier.fit(train_embedding.cpu().numpy(), labels[train_seeds].cpu().numpy()) + xgb_pred = classifier.predict_proba(test_embedding.cpu().numpy()) + acc, f_1, precision, recall, roc_auc, pr_auc, average_precision, _, _ = get_metrics( + xgb_pred, labels[inductive_idx].cpu().numpy(), name='RGCN_XGB') + metrics_result += [{ + 'model': 'RGCN_XGB', + 'acc': acc, + 'f1': f_1, + 'precision': precision, + 'recall': recall, + 'roc_auc': roc_auc, + 'pr_auc': pr_auc, + 'ap': average_precision + }] + + # Save model + pd.DataFrame(metrics_result).to_csv(output_file) + save_model(whole_graph, model, hyperparameters, classifier, model_dir) + + end = timer() + print(end - start) if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--training-data", required=True, help="CSV with fraud_label") - parser.add_argument("--validation-data", required=False, help="CSV with fraud_label") - parser.add_argument("--epochs", help="Number of epochs", type=int, default=10) - parser.add_argument("--node_type", required=False, help="Target node type", default="transaction") - parser.add_argument("--output-xgb", required=False, help="output file to save xgboost model") - parser.add_argument("--output-hinsage", required=False, help="output file to save GraphHinSage model") - parser.add_argument("--save_model", type=bool, default=False, help="Save models to give filenames") - parser.add_argument("--embedding_size", required=False, default=64, help="output file to save new model") - - args = parser.parse_args() - - # Global parameters: - embedding_size = int(args.embedding_size) - epochs = int(args.epochs) - EMBEDDING_NODE_TYPE = str(args.node_type) - num_samples = [2, 32] - - main() + train_model() diff --git a/models/validation-inference-scripts/fraud-detection-models/inference.py b/models/validation-inference-scripts/fraud-detection-models/inference.py index 039cc1f714..02d470eaf5 100644 --- a/models/validation-inference-scripts/fraud-detection-models/inference.py +++ b/models/validation-inference-scripts/fraud-detection-models/inference.py @@ -13,106 +13,80 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -Example Usage: -python inference.py --graph-data ../../datasets/training-data/fraud-detection-training-data.csv \ - --validation-data ../../datasets/validation-data/fraud-detection-validation-data.csv \ - --model-xgb model/xgb-model.pt --model-hinsage model/hinsage-model.pt --output out.txt +Usage example: +python inference.py --training-data ../../datasets/training-data/fraud-detection-training-data.csv\ + --validation-data ../../datasets/validation-data/fraud-detection-validation-data.csv\ + --model-dir ../../fraud-detection-models\ + --output-file out.txt --model-type HinSAGE + +Note: This script uses symbolink of model file located at +../../../examples/gnn_fraud_detection_pipeline/stages/model.py """ -import argparse +import os -import networkx as nx +import click +import numpy as np import pandas as pd -import tensorflow as tf -from stellargraph import StellarGraph -from stellargraph.mapper import HinSAGENodeGenerator - +import torch +from model import HeteroRGCN +from model import HinSAGE +from model import build_fsi_graph +from model import load_model +from model import prepare_data + +import cudf as cf from cuml import ForestInference - -def graph_construction(nodes, edges, node_features): - g_nx = nx.Graph() - - # add nodes - for key, values in nodes.items(): - g_nx.add_nodes_from(values, ntype=key) - # add edges - for edge in edges: - g_nx.add_edges_from(edge) - - return StellarGraph(g_nx, node_type_name="ntype", node_features=node_features) - - -def build_graph_features(dataset): - # Train data - transaction_node_data = dataset.drop(["client_node", "merchant_node", "fraud_label", "index"], axis=1) - client_node_data = pd.DataFrame([1] * len(dataset.client_node.unique())).set_index(dataset.client_node.unique()) - merchant_node_data = pd.DataFrame([1] * len(dataset.merchant_node.unique())).set_index( - dataset.merchant_node.unique()) - - nodes = {"client": dataset.client_node, "merchant": dataset.merchant_node, "transaction": dataset.index} - edges = [zip(dataset.client_node, dataset.index), zip(dataset.merchant_node, dataset.index)] - features = {"transaction": transaction_node_data, 'client': client_node_data, 'merchant': merchant_node_data} - - graph = graph_construction(nodes, edges, features) # GraphConstruction(nodes, edges, features) - # S = graph.get_stellargraph() - return graph - - -def inductive_step_hinsage(S, trained_model, inductive_node_identifiers, batch_size): - # perform inductive learning from trained graph model - - num_samples = [2, 32] - # The mapper feeds data from sampled subgraph to HinSAGE model - generator = HinSAGENodeGenerator(S, batch_size, num_samples, head_node_type="transaction") - test_gen_not_shuffled = generator.flow(inductive_node_identifiers, shuffle=False) - - inductive_emb = trained_model.predict(test_gen_not_shuffled, verbose=1) - inductive_emb = pd.DataFrame(inductive_emb, index=inductive_node_identifiers) - - return inductive_emb +np.random.seed(1001) +torch.manual_seed(1001) -def infer(model_xgb, model_hinsage, graph_data, node_identifier, output): +@click.command() +@click.option('--training-data', help="Path to training data for graph structure.", default="data/training.csv") +@click.option('--validation-data', help="Path to validation data", default="data/validation.csv") +@click.option('--model-dir', help="path to model directory", default="modeldir") +@click.option('--target-node', help="Target node", default="transaction") +@click.option('--output-file', help="Path to csv inference result", default="out.csv") +@click.option('--model-type', help="Model type either RGCN/Graphsage", default="RGCN") +def main(training_data, validation_data, model_dir, target_node, output_file, model_type): - # Build graph structure. + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + meta_cols = ["client_node", "merchant_node", "index"] + if model_type == "RGCN": + gnn_model = HeteroRGCN + else: + gnn_model = HinSAGE - graph = build_graph_features(graph_data) + # prepare data + training_data, validation_data = cf.read_csv(training_data), cf.read_csv(validation_data) + _, _, _, test_index, _, all_data = prepare_data(training_data, validation_data) - # Load XGBoost & GraphSAGE model - xgb_model = ForestInference.load(model_xgb, output_class=True) - hgs_model = tf.keras.models.load_model(model_hinsage) + # build graph structure + input_graph, feature_tensors = build_fsi_graph(all_data, meta_cols) + test_index = torch.from_dlpack(test_index.values.toDlpack()).long() - inductive_embedding = inductive_step_hinsage(graph, hgs_model, node_identifier, batch_size=5) + # Load graph model, return only the gnn model + model, _, _ = load_model(model_dir, gnn_model=gnn_model) - # prediction - prediction = xgb_model.predict_proba(inductive_embedding)[:, 1] - result = pd.DataFrame(node_identifier, columns=['node_id']) - result['prediction'] = prediction - result.to_csv(output, index=False) - return result + # Load XGBoost model + xgb_model = ForestInference.load(os.path.join(model_dir, 'xgb.pt'), output_class=True) + model = model.to(device) + input_graph = input_graph.to(device) -def main(): - graph_data = pd.read_csv(args.graph_data) - val_data = pd.read_csv(args.validation_data) - graph_data = pd.concat([graph_data, val_data]) - graph_data = graph_data.set_index(graph_data['index']) + # Perform inference + test_embedding, test_seeds = model.inference(input_graph, feature_tensors.float(), test_index, target_node) - infer(args.model_xgb, - args.model_hinsage, - graph_data=graph_data, - node_identifier=list(val_data['index']), - output=args.output) + # collect result . XGBoost predict_proba(test_embedding)[:, 1] + # indicates probability score of negative class using XGBoost. + pred_score = xgb_model.predict_proba(test_embedding)[:, 1] + df_result = pd.DataFrame(test_seeds.cpu(), columns=['node_id']) + df_result['score'] = pred_score.get() + df_result.to_csv(output_file, index=False) + print(df_result) -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--validation-data", required=False, help="Labelled data in CSV format") - parser.add_argument("--model-hinsage", required=True, help="trained hinsage model") - parser.add_argument("--model-xgb", required=True, help="trained xgb model") - parser.add_argument("--graph-data", help="Training dataset for graph structure", required=True) - parser.add_argument("--output", required=True, help="output filename") - args = parser.parse_args() -main() +if __name__ == '__main__': + main() diff --git a/models/validation-inference-scripts/fraud-detection-models/model.py b/models/validation-inference-scripts/fraud-detection-models/model.py new file mode 120000 index 0000000000..96ac13f676 --- /dev/null +++ b/models/validation-inference-scripts/fraud-detection-models/model.py @@ -0,0 +1 @@ +../../../examples/gnn_fraud_detection_pipeline/stages/model.py \ No newline at end of file diff --git a/models/validation-inference-scripts/fraud-detection-models/requirements.txt b/models/validation-inference-scripts/fraud-detection-models/requirements.txt deleted file mode 100644 index 70d455c69a..0000000000 --- a/models/validation-inference-scripts/fraud-detection-models/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -cudf==0+untagged.1.gfc341af -cuml==0+untagged.1.g835a9ae -dateparser==1.1.0 -matplotlib==3.4.3 -networkx==2.6.3 -numpy==1.22.0 -pandas==1.3.3 -scikit_learn==1.0.2 -stellargraph==1.2.1 -tensorflow==2.8.0 -tensorflow_gpu==2.7.0 -xgboost==1.4.2 diff --git a/models/validation-inference-scripts/fraud-detection-models/requirements.yml b/models/validation-inference-scripts/fraud-detection-models/requirements.yml new file mode 100644 index 0000000000..7fe973ff1d --- /dev/null +++ b/models/validation-inference-scripts/fraud-detection-models/requirements.yml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +channels: + - rapidsai + - nvidia + - conda-forge +dependencies: + - click==8.1.3 + - cuml=23.06 + - dgl==1.0.2+cu118 + - numpy==1.23.5 + - pandas==1.5.3 + - scikit_learn==1.2.2 + - torch==2.0.0+cu118 + - torchmetrics==0.11.4 + - tqdm==4.65.0 + - xgboost==1.7.1 diff --git a/morpheus.code-workspace b/morpheus.code-workspace index d4e71cc79c..8a10eda624 100644 --- a/morpheus.code-workspace +++ b/morpheus.code-workspace @@ -560,7 +560,39 @@ "loadAll": false }, "type": "cppdbg" - } + }, + { + "name": "Python: GNN DGL inference", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/examples/gnn_fraud_detection_pipeline/run.py", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/examples/gnn_fraud_detection_pipeline", + "justMyCode": false, + "args": [ + "--input_file=validation.csv", + "--training_file=training.csv", + "--output_file=result.csv", + "--model_dir=model" + ] + }, + { + "name": "Python: GNN model training", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/models/training-tuning-scripts/fraud-detection-models/training.py", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/models/training-tuning-scripts/fraud-detection-models", + "justMyCode": false, + "args": [ + "--training-data=${workspaceFolder}/examples/gnn_fraud_detection_pipeline/training.csv", + "--validation-data=${workspaceFolder}/examples/gnn_fraud_detection_pipeline/validation.csv", + "--output-file=result.csv", + "--model-dir=models", + "--epochs=5" + ] + } + ] }, "settings": { diff --git a/pyproject.toml b/pyproject.toml index e0dbbe46b7..7a544ca988 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,17 +41,14 @@ ignore_patterns = [ "**/*.pxd", ] - [tool.mypy] # Allow None for argument default values implicit_optional = true - [tool.pyright] # Allow None for argument default values strictParameterNoneValue = false - [tool.interrogate] exclude = ["morpheus/cli"] ignore-init-method = true @@ -624,8 +621,8 @@ contextmanager-decorators = ["contextlib.contextmanager"] # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -# Add pytorch members to the list of dynamically set members. Only until pylint 3.0 supports pyi files -generated-members = "torch.*" +# Add pytorch & dgl members to the list of dynamically set members. Only until pylint 3.0 supports pyi files +generated-members = "torch.*,dgl.*" # Tells whether missing members accessed in mixin class should be ignored. A # class is considered mixin if its name matches the mixin-class-rgx option. diff --git a/tests/_utils/__init__.py b/tests/_utils/__init__.py index 2cbd0df99e..c8c621172b 100644 --- a/tests/_utils/__init__.py +++ b/tests/_utils/__init__.py @@ -17,6 +17,7 @@ import collections import json import os +import sys import time import types import typing @@ -197,3 +198,15 @@ def async_infer(callback=None, **_): callback(mock_infer_result, None) return async_infer + + +def remove_module(mod_to_remove: str): + """ + Remove a module, and all sub-modules from `sys.modules`. This is needed when testing examples which may import + modules with common names such as `stages` which need to be removed from `sys.modules` before running tests for + another example which might also contain its own `stages` module. + """ + mod_prefix = f"{mod_to_remove}." + for mod_name in list(sys.modules.keys()): + if mod_name == mod_to_remove or mod_name.startswith(mod_prefix): + del sys.modules[mod_name] diff --git a/tests/examples/gnn_fraud_detection_pipeline/conftest.py b/tests/examples/gnn_fraud_detection_pipeline/conftest.py index 8384d892ae..a3628f20f2 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/conftest.py +++ b/tests/examples/gnn_fraud_detection_pipeline/conftest.py @@ -20,36 +20,29 @@ from _utils import TEST_DIRS from _utils import import_or_skip +from _utils import remove_module SKIP_REASON = ("Tests for the gnn_fraud_detection_pipeline example require a number of packages not installed in the " "Morpheus development environment. See `examples/gnn_fraud_detection_pipeline/README.md` for details on " "installing these additional dependencies") -@pytest.fixture(autouse=True, scope='session') -def stellargraph(fail_missing: bool): +@pytest.fixture(name="dgl", autouse=True, scope='session') +def dgl_fixture(fail_missing: bool): """ - All of the tests in this subdir require stellargraph + All of the tests in this subdir require dgl """ - yield import_or_skip("stellargraph", reason=SKIP_REASON, fail_missing=fail_missing) + yield import_or_skip("dgl", reason=SKIP_REASON, fail_missing=fail_missing) -@pytest.fixture(autouse=True, scope='session') -def cuml(fail_missing: bool): +@pytest.fixture(name="cuml", autouse=True, scope='session') +def cuml_fixture(fail_missing: bool): """ All of the tests in this subdir require cuml """ yield import_or_skip("cuml", reason=SKIP_REASON, fail_missing=fail_missing) -@pytest.fixture(autouse=True, scope='session') -def tensorflow(fail_missing: bool): - """ - All of the tests in this subdir require tensorflow - """ - yield import_or_skip("tensorflow", reason=SKIP_REASON, fail_missing=fail_missing) - - @pytest.fixture(name="config") def config_fixture(config): """ @@ -60,40 +53,61 @@ def config_fixture(config): yield config +@pytest.fixture(name="manual_seed", scope="function") +def manual_seed_fixture(manual_seed): + """ + Extends the base `manual_seed` fixture to also set the seed for dgl, ensuring deterministic results in tests + """ + import dgl + + def seed_fn(seed=42): + manual_seed(seed) + dgl.seed(seed) + + seed_fn() + yield seed_fn + + @pytest.fixture(name="example_dir") def example_dir_fixture(): yield os.path.join(TEST_DIRS.examples_dir, 'gnn_fraud_detection_pipeline') -@pytest.fixture -def training_file(example_dir: str): +@pytest.fixture(name="training_file") +def training_file_fixture(example_dir: str): yield os.path.join(example_dir, 'training.csv') -@pytest.fixture -def hinsage_model(example_dir: str): - yield os.path.join(example_dir, 'model/hinsage-model.pt') +@pytest.fixture(name="model_dir") +def model_dir_fixture(example_dir: str): + yield os.path.join(example_dir, 'model') -@pytest.fixture -def xgb_model(example_dir: str): - yield os.path.join(example_dir, 'model/xgb-model.pt') +@pytest.fixture(name="xgb_model") +def xgb_model_fixture(model_dir: str): + yield os.path.join(model_dir, 'xgb.pt') # Some of the code inside gnn_fraud_detection_pipeline performs some relative imports in the form of: # from .mod import Class # For this reason we need to ensure that the examples dir is in the sys.path first -@pytest.fixture(name="gnn_fraud_detection_pipeline") -def gnn_fraud_detection_pipeline_fixture( - restore_sys_path, # pylint: disable=unused-argument - reset_plugins): # pylint: disable=unused-argument - sys.path.append(TEST_DIRS.examples_dir) - import gnn_fraud_detection_pipeline - yield gnn_fraud_detection_pipeline +@pytest.mark.usefixtures("restore_sys_path", "reset_plugins") +@pytest.fixture(name="ex_in_sys_path", autouse=True) +def ex_in_sys_path_fixture(example_dir: str): + sys.path.insert(0, example_dir) + + +@pytest.fixture(autouse=True) +def reset_modules(): + """ + Other examples have a stages module, ensure it is un-imported after running tests in this subdir + """ + yield + remove_module('stages') -@pytest.fixture -def test_data(): +@pytest.fixture(name="test_data") +def test_data_fixture(): """ Construct test data, a small DF of 10 rows which we will build a graph from The nodes in our graph will be the unique values from each of our three columns, and the index is also @@ -104,10 +118,11 @@ def test_data(): thus we should expect 20 edges, although 2697 is duplicated in the client_node column we should expect two unique edges for each entry (2697, 14) & (2697, 91) """ - import pandas as pd + import cudf index = [2, 14, 16, 26, 41, 42, 70, 91, 93, 95] - client_data = [795, 2697, 5531, 415, 2580, 3551, 6547, 2697, 3503, 7173] - merchant_data = [8567, 4609, 2781, 7844, 629, 6915, 7071, 570, 2446, 8110] + + client_data = [795.0, 2697.0, 5531.0, 415.0, 2580.0, 3551.0, 6547.0, 2697.0, 3503.0, 7173.0] + merchant_data = [8567.0, 4609.0, 2781.0, 7844.0, 629.0, 6915.0, 7071.0, 570.0, 2446.0, 8110.0] df_data = { 'index': index, @@ -120,24 +135,37 @@ def test_data(): for i in range(1000, 1113): # these two values are skipped, apparently place-holders for client_node & merchant_node if i not in (1002, 1003): - df_data[str(i)] = [0 for _ in range(len(index))] + df_data[str(i)] = [0.0 for _ in range(len(index))] + + df = cudf.DataFrame(df_data, index=index) + + # Create indexed nodeId + meta_cols = ['index', 'client_node', 'merchant_node'] + for col in meta_cols: + df[col] = cudf.CategoricalIndex(df[col]).codes + df.index = df['index'] - df = pd.DataFrame(df_data, index=index) + # Collect expected nodes, since hetero nodes could share same index + # We use dict of node_name:index + expected_nodes = {} + for col in meta_cols: + expected_nodes[col] = set(df[col].to_arrow().tolist()) - expected_nodes = set(index + client_data + merchant_data) - assert len(expected_nodes) == 29 # ensuring test data & assumptions are correct + # ensuring test data & assumptions are correct + assert sum(len(nodes) for _, nodes in expected_nodes.items()) == 29 - expected_edges = set() - for data in (client_data, merchant_data): - for (i, val) in enumerate(data): - expected_edges.add((val, index[i])) + expected_edges = {'buy': [], 'sell': []} + for i in range(df.shape[0]): + for key, val in {'buy': 'client_node', 'sell': 'merchant_node'}.items(): + expected_edges[key].append([df[val].iloc[i], i]) - assert len(expected_edges) == 20 # ensuring test data & assumptions are correct + # ensuring test data & assumptions are correct + assert sum(len(edges) for _, edges in expected_edges.items()) == 20 yield { - "index": index, - "client_data": client_data, - "merchant_data": merchant_data, + "index": df['index'].to_arrow().tolist(), + "client_data": df['client_node'].to_arrow().tolist(), + "merchant_data": df['merchant_node'].to_arrow().tolist(), "df": df, "expected_nodes": expected_nodes, "expected_edges": expected_edges diff --git a/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py b/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py index 20c50d5dbf..ca484dc959 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py +++ b/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py @@ -21,29 +21,22 @@ from morpheus.config import Config from morpheus.messages import MessageMeta +# pylint: disable=no-name-in-module + @pytest.mark.use_python class TestClassificationStage: - def test_constructor( - self, - config: Config, - xgb_model: str, - gnn_fraud_detection_pipeline: types.ModuleType, # pylint: disable=unused-argument - cuml: types.ModuleType): - from gnn_fraud_detection_pipeline.stages.classification_stage import ClassificationStage + def test_constructor(self, config: Config, xgb_model: str, cuml: types.ModuleType): + from stages.classification_stage import ClassificationStage stage = ClassificationStage(config, xgb_model) assert isinstance(stage._xgb_model, cuml.ForestInference) - def test_process_message( - self, - config: Config, - xgb_model: str, - gnn_fraud_detection_pipeline: types.ModuleType, # pylint: disable=unused-argument - dataset_cudf: DatasetManager): - from gnn_fraud_detection_pipeline.stages.classification_stage import ClassificationStage - from gnn_fraud_detection_pipeline.stages.graph_sage_stage import GraphSAGEMultiMessage + @pytest.mark.usefixtures("manual_seed") + def test_process_message(self, config: Config, xgb_model: str, dataset_cudf: DatasetManager): + from stages.classification_stage import ClassificationStage + from stages.graph_sage_stage import GraphSAGEMultiMessage df = dataset_cudf['examples/gnn_fraud_detection_pipeline/inductive_emb.csv'] df.rename(lambda x: f"ind_emb_{x}", axis=1, inplace=True) diff --git a/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py b/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py index 27877c307b..11db2ed799 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py +++ b/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py @@ -13,107 +13,67 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import types -import typing from io import StringIO -import pandas as pd import pytest +import torch import cudf -from _utils import TEST_DIRS from morpheus.config import Config from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage +# pylint: disable=no-name-in-module + @pytest.mark.use_python -@pytest.mark.import_mod( - [os.path.join(TEST_DIRS.examples_dir, 'gnn_fraud_detection_pipeline/stages/graph_construction_stage.py')]) class TestGraphConstructionStage: - def test_constructor(self, config: Config, training_file: str, import_mod: typing.List[types.ModuleType]): - graph_construction_stage = import_mod[0] - stage = graph_construction_stage.FraudGraphConstructionStage(config, training_file) + def test_constructor(self, config: Config, training_file: str): + from stages.graph_construction_stage import FraudGraphConstructionStage + stage = FraudGraphConstructionStage(config, training_file) assert isinstance(stage._training_data, cudf.DataFrame) # The training datafile contains many more columns than this, but these are the four columns # that are depended upon in the code assert {'client_node', 'index', 'fraud_label', 'merchant_node'}.issubset(stage._column_names) - def _check_graph( - self, - stellargraph: types.ModuleType, - graph: "stellargraph.StellarGraph", # noqa: F821 - expected_nodes, - expected_edges): - assert isinstance(graph, stellargraph.StellarGraph) - graph.check_graph_for_ml(features=True, expensive_check=True) # this will raise if it doesn't pass - assert not graph.is_directed() - - nodes = graph.nodes() - assert set(nodes) == expected_nodes - - edges = graph.edges() - assert set(edges) == expected_edges - - def test_graph_construction(self, - import_mod: typing.List[types.ModuleType], - stellargraph: types.ModuleType, - test_data: dict): - graph_construction_stage = import_mod[0] - df = test_data['df'] - - client_features = pd.DataFrame({0: 1}, index=list(set(test_data['client_data']))) - merchant_features = pd.DataFrame({0: 1}, index=test_data['merchant_data']) - - # Call _graph_construction - graph = graph_construction_stage.FraudGraphConstructionStage._graph_construction( - nodes={ - 'client': df.client_node, 'merchant': df.merchant_node, 'transaction': df.index - }, - edges=[ - zip(df.client_node, df.index), - zip(df.merchant_node, df.index), - ], - node_features={ - "transaction": df[['client_node', 'merchant_node']], - "client": client_features, - "merchant": merchant_features - }) - - self._check_graph(stellargraph, graph, test_data['expected_nodes'], test_data['expected_edges']) - - def test_build_graph_features(self, - import_mod: typing.List[types.ModuleType], - stellargraph: types.ModuleType, - test_data: dict): - graph_construction_stage = import_mod[0] - graph = graph_construction_stage.FraudGraphConstructionStage._build_graph_features(test_data['df']) - self._check_graph(stellargraph, graph, test_data['expected_nodes'], test_data['expected_edges']) - - def test_process_message(self, - config: Config, - import_mod: typing.List[types.ModuleType], - stellargraph: types.ModuleType, - test_data: dict): - graph_construction_stage = import_mod[0] + def test_process_message(self, dgl: types.ModuleType, config: Config, test_data: dict): + from stages import graph_construction_stage df = test_data['df'] + expected_nodes = test_data['expected_nodes'] + expected_edges = test_data['expected_edges'] # The stage wants a csv file from the first 5 rows - training_data = StringIO(df[0:5].to_csv(index=False)) + training_data = StringIO(df.head(5).to_csv(index=False)) stage = graph_construction_stage.FraudGraphConstructionStage(config, training_data) # Since we used the first 5 rows as the training data, send the second 5 as inference data - meta = MessageMeta(cudf.DataFrame(df)) - multi_mesg = MultiMessage(meta=meta, mess_offset=5, mess_count=5) - fgmm = stage._process_message(multi_mesg) + meta = MessageMeta(cudf.DataFrame(df).tail(5)) + multi_msg = MultiMessage(meta=meta) + fgmm = stage._process_message(multi_msg) assert isinstance(fgmm, graph_construction_stage.FraudGraphMultiMessage) assert fgmm.meta is meta - assert fgmm.mess_offset == 5 + assert fgmm.mess_offset == 0 assert fgmm.mess_count == 5 - self._check_graph(stellargraph, fgmm.graph, test_data['expected_nodes'], test_data['expected_edges']) + assert isinstance(fgmm.graph, dgl.DGLGraph) + + # Since the graph has a reverse edge for each edge, one edge comparison is enough. + buy_edges = fgmm.graph.edges(etype='buy') + sell_edges = fgmm.graph.edges(etype='sell') + + # expected edges, convert [(u,v)] format to [u, v] of DGL edge format. + exp_buy_edges = [torch.LongTensor(e).cuda() for e in zip(*expected_edges['buy'])] + exp_sell_edges = [torch.LongTensor(e).cuda() for e in zip(*expected_edges['sell'])] + + # Compare all edges types agree. + assert all(exp_buy_edges[0] == buy_edges[0]) & all(exp_buy_edges[1] == buy_edges[1]) + assert all(exp_sell_edges[0] == sell_edges[0]) & all(exp_sell_edges[1] == sell_edges[1]) + + # Compare nodes. + for node in ['client', 'merchant']: + assert fgmm.graph.nodes(node).tolist() == list(expected_nodes[node + "_node"]) diff --git a/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py b/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py index 183a2765f9..e9fc5f01c6 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py +++ b/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import types - import pytest import cudf @@ -22,85 +20,53 @@ from _utils.dataset_manager import DatasetManager from morpheus.config import Config from morpheus.messages import MessageMeta +from morpheus.messages import MultiMessage + +# pylint: disable=no-name-in-module @pytest.mark.use_python class TestGraphSageStage: - def test_constructor( - self, - config: Config, - hinsage_model: str, - gnn_fraud_detection_pipeline: types.ModuleType, # pylint: disable=unused-argument - tensorflow): - from gnn_fraud_detection_pipeline.stages.graph_sage_stage import GraphSAGEStage - stage = GraphSAGEStage(config, - model_hinsage_file=hinsage_model, - batch_size=10, - sample_size=[4, 64], - record_id="test_id", - target_node="test_node") + def test_constructor(self, config: Config, model_dir: str): + from stages.graph_sage_stage import GraphSAGEStage + from stages.model import HinSAGE + stage = GraphSAGEStage(config, model_dir=model_dir, batch_size=10, record_id="test_id", target_node="test_node") - assert isinstance(stage._keras_model, tensorflow.keras.models.Model) + assert isinstance(stage._dgl_model, HinSAGE) assert stage._batch_size == 10 - assert stage._sample_size == [4, 64] assert stage._record_id == "test_id" assert stage._target_node == "test_node" - def test_inductive_step_hinsage( - self, - config: Config, - hinsage_model: str, - gnn_fraud_detection_pipeline: types.ModuleType, # pylint: disable=unused-argument - test_data: dict, - dataset_pandas: DatasetManager): - from gnn_fraud_detection_pipeline.stages.graph_construction_stage import FraudGraphConstructionStage - from gnn_fraud_detection_pipeline.stages.graph_sage_stage import GraphSAGEStage - - # The column names in the saved test data will be strings, in the results they will be ints - expected_df = dataset_pandas['examples/gnn_fraud_detection_pipeline/inductive_emb.csv'] - expected_df.rename(lambda x: int(x), axis=1, inplace=True) - - df = test_data['df'] - - graph = FraudGraphConstructionStage._build_graph_features(df) - - stage = GraphSAGEStage(config, model_hinsage_file=hinsage_model) - results = stage._inductive_step_hinsage(graph, stage._keras_model, test_data['index']) - - assert isinstance(results, cudf.DataFrame) - assert results.index.to_arrow().to_pylist() == test_data['index'] - dataset_pandas.assert_compare_df(results, expected_df) - - def test_process_message( - self, - config: Config, - hinsage_model: str, - gnn_fraud_detection_pipeline: types.ModuleType, # pylint: disable=unused-argument - test_data: dict, - dataset_pandas: DatasetManager): - from gnn_fraud_detection_pipeline.stages.graph_construction_stage import FraudGraphConstructionStage - from gnn_fraud_detection_pipeline.stages.graph_construction_stage import FraudGraphMultiMessage - from gnn_fraud_detection_pipeline.stages.graph_sage_stage import GraphSAGEMultiMessage - from gnn_fraud_detection_pipeline.stages.graph_sage_stage import GraphSAGEStage + @pytest.mark.usefixtures("manual_seed") + def test_process_message(self, + config: Config, + training_file: str, + model_dir: str, + test_data: dict, + dataset_pandas: DatasetManager): + from stages.graph_construction_stage import FraudGraphConstructionStage + from stages.graph_sage_stage import GraphSAGEMultiMessage + from stages.graph_sage_stage import GraphSAGEStage expected_df = dataset_pandas['examples/gnn_fraud_detection_pipeline/inductive_emb.csv'] - expected_df.rename(lambda x: f"ind_emb_{x}", axis=1, inplace=True) df = test_data['df'] meta = MessageMeta(cudf.DataFrame(df)) - graph = FraudGraphConstructionStage._build_graph_features(df) - msg = FraudGraphMultiMessage(meta=meta, graph=graph) + multi_msg = MultiMessage(meta=meta) + construction_stage = FraudGraphConstructionStage(config, training_file) + fgmm_msg = construction_stage._process_message(multi_msg) - stage = GraphSAGEStage(config, model_hinsage_file=hinsage_model) - results = stage._process_message(msg) + stage = GraphSAGEStage(config, model_dir=model_dir) + results = stage._process_message(fgmm_msg) assert isinstance(results, GraphSAGEMultiMessage) assert results.meta is meta assert results.mess_offset == 0 assert results.mess_count == len(df) assert results.node_identifiers == test_data['index'] - assert sorted(results.inductive_embedding_column_names) == sorted(expected_df.columns) - ind_emb_df = results.get_meta(results.inductive_embedding_column_names) + cols = results.inductive_embedding_column_names + ['index'] + assert sorted(cols) == sorted(expected_df.columns) + ind_emb_df = results.get_meta(cols) dataset_pandas.assert_compare_df(ind_emb_df.to_pandas(), expected_df) diff --git a/tests/examples/ransomware_detection/conftest.py b/tests/examples/ransomware_detection/conftest.py index 2ee86fd198..8f1d901f6b 100644 --- a/tests/examples/ransomware_detection/conftest.py +++ b/tests/examples/ransomware_detection/conftest.py @@ -21,6 +21,7 @@ from _utils import TEST_DIRS from _utils import import_or_skip +from _utils import remove_module # pylint: disable=redefined-outer-name @@ -73,8 +74,18 @@ def interested_plugins(): # Some of the code inside ransomware_detection performs imports in the form of: # from common.... # For this reason we need to ensure that the examples/ransomware_detection dir is in the sys.path first -# pylint: disable=unused-argument @pytest.fixture(autouse=True) -@pytest.mark.usefixtures("request", "restore_sys_path", "reset_plugins") -def ransomware_detection_in_sys_path(example_dir): - sys.path.append(example_dir) +@pytest.mark.usefixtures("restore_sys_path", "reset_plugins") +def ransomware_detection_in_sys_path(example_dir: str): + sys.path.insert(0, example_dir) + + +@pytest.fixture(autouse=True) +def reset_modules(): + """ + Other examples could potentially have modules with the same name as the modules in this example. Ensure any + modules imported by these tests are removed from sys.modules after the test is completed. + """ + yield + for remove_mod in ('common', 'stages'): + remove_module(remove_mod) diff --git a/tests/tests_data/examples/gnn_fraud_detection_pipeline/inductive_emb.csv b/tests/tests_data/examples/gnn_fraud_detection_pipeline/inductive_emb.csv index 3c8eae2372..9ae8600dd2 100644 --- a/tests/tests_data/examples/gnn_fraud_detection_pipeline/inductive_emb.csv +++ b/tests/tests_data/examples/gnn_fraud_detection_pipeline/inductive_emb.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:278c304032df75c9c2e13d0367c08ad4d652fb77a86a50ea7b1d26c37e282a76 -size 8142 +oid sha256:e5f824196fdc9a0e027d65369ae7f73e44fe2986058cc1af46c3c04d62217f74 +size 8641 diff --git a/tests/tests_data/examples/gnn_fraud_detection_pipeline/predictions.csv b/tests/tests_data/examples/gnn_fraud_detection_pipeline/predictions.csv index 87256ef044..ad9949caf5 100644 --- a/tests/tests_data/examples/gnn_fraud_detection_pipeline/predictions.csv +++ b/tests/tests_data/examples/gnn_fraud_detection_pipeline/predictions.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4baf153412823c07ebc35684ef88e2237aeef159ba0857dd98ef80190c639e44 -size 168 +oid sha256:95f80549b2362618359519c6bcd140125397ad3a60858baf84436798ef83f1dd +size 164