From 0dd0e9207b522dade90cc6ab6c3fbe3c3ca0290c Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 6 Dec 2023 23:01:31 +0100 Subject: [PATCH 1/2] misc: add force_driven graph example --- examples/{rerun => rerun_example}/.gitignore | 0 .../{rerun => rerun_example}/dna_example.py | 0 .../force_driven_lockfile_graph.py | 109 ++++++ examples/{rerun => rerun_example}/pixi.lock | 345 ++++++++++++++++-- examples/{rerun => rerun_example}/pixi.toml | 8 +- 5 files changed, 420 insertions(+), 42 deletions(-) rename examples/{rerun => rerun_example}/.gitignore (100%) rename examples/{rerun => rerun_example}/dna_example.py (100%) create mode 100644 examples/rerun_example/force_driven_lockfile_graph.py rename examples/{rerun => rerun_example}/pixi.lock (96%) rename examples/{rerun => rerun_example}/pixi.toml (64%) diff --git a/examples/rerun/.gitignore b/examples/rerun_example/.gitignore similarity index 100% rename from examples/rerun/.gitignore rename to examples/rerun_example/.gitignore diff --git a/examples/rerun/dna_example.py b/examples/rerun_example/dna_example.py similarity index 100% rename from examples/rerun/dna_example.py rename to examples/rerun_example/dna_example.py diff --git a/examples/rerun_example/force_driven_lockfile_graph.py b/examples/rerun_example/force_driven_lockfile_graph.py new file mode 100644 index 000000000..45315c2e3 --- /dev/null +++ b/examples/rerun_example/force_driven_lockfile_graph.py @@ -0,0 +1,109 @@ +import rerun as rr +import networkx as nx +import yaml +import numpy as np +import hashlib +import sys + +# Give relative path or default to local pixi.lock +lockfile_path = sys.argv[1] if len(sys.argv) > 1 else 'pixi.lock' + +with open(lockfile_path, 'r') as file: + lockfile_data = yaml.safe_load(file) + +package_data = lockfile_data['package'] +package_names = [package['name'] for package in package_data] + +graph = nx.DiGraph() +for package in package_data: + package_name = package['name'] + dependencies = package.get('dependencies', []) + graph.add_node(package_name) + for i, dep in enumerate(dependencies): + graph.add_edge(package_name, dep.split(" ")[0]) + +rr.init("fdg", spawn=True) +rr.connect() + +# Force-Directed Simulation Parameters +iterations = 100 +repulsive_force = 0.04 +attractive_force = 0.002 + + +def hash_string_to_int(string): + return int(hashlib.sha256(string.encode('utf-8')).hexdigest(), 16) % (10 ** 8) + + +# Memoization dictionary +color_cache = {} + + +# Function to get color +def get_color_for_node(node): + if node not in color_cache: + np.random.seed(hash_string_to_int(node)) + color_cache[node] = np.random.rand(3) # Generate and store color + return color_cache[node] + + +def apply_forces_and_log(graph, pos): + damping = 0.9 + max_force = 10 + degree_scale = 0.9 # Scale factor for degree-based forces + + for iteration in range(iterations): + force = {node: np.zeros(3) for node in graph} + + # Degree-based repulsive forces + for i, node1 in enumerate(graph): + for node2 in list(graph)[i + 1:]: + diff = pos[node1] - pos[node2] + dist = np.linalg.norm(diff) + 1e-9 # Avoid divide-by-zero + degree_factor = ( + (graph.degree(node1) + graph.degree(node2)) * degree_scale) + repel = repulsive_force * degree_factor / dist ** 2 + force_vector = repel * diff # / dist + force[node1] += np.clip(force_vector, -max_force, max_force) + force[node2] -= np.clip(force_vector, -max_force, max_force) + + # Degree-based attractive forces + for edge in graph.edges(): + u, v = edge + diff = pos[u] - pos[v] + dist = np.linalg.norm(diff) + if dist > 0: + degree_factor = (graph.degree(u) + graph.degree(v)) * degree_scale + attract = (attractive_force * dist ** 2) / degree_factor + force[u] -= attract * diff / dist + force[v] += attract * diff / dist + + # Update positions with damping + for node in graph: + pos[node] += force[node] * damping + position = np.array(pos[node]) + color = get_color_for_node(node) # Retrieve color, memoized + rr.log(f"graph_nodes/{node}", + rr.Points3D([position], + colors=[color], + radii=max(graph.degree(node) / 20, 0.5)), + rr.AnyValues(node)) + + edges_array = np.array([[pos[u], pos[v]] for u, v in graph.edges()]) + + # Log the edges array + rr.log("graph_nodes/graph_edges", + rr.LineStrips3D(edges_array, radii=0.02, colors=[1, 1, 1, 0.1])) + + return pos + + +# Identify the node with the highest degree +central_node = max(graph.degree, key=lambda x: x[1])[0] + +# Initial positions with the central node at the center +initial_pos = nx.spring_layout(graph, dim=3) +initial_pos[central_node] = np.array([0.5, 0.5, 0.5]) # Center position + +# Apply the force-directed simulation +final_pos = apply_forces_and_log(graph, initial_pos) diff --git a/examples/rerun/pixi.lock b/examples/rerun_example/pixi.lock similarity index 96% rename from examples/rerun/pixi.lock rename to examples/rerun_example/pixi.lock index 69bc0bdc5..8958e420d 100644 --- a/examples/rerun/pixi.lock +++ b/examples/rerun_example/pixi.lock @@ -1,4 +1,4 @@ -version: 2 +version: 3 metadata: content_hash: linux-64: e90c2ee71ad70fc0a1c8302029533a7d1498f2bffcd0eaa8d2934700e775dc1d @@ -5260,24 +5260,124 @@ package: license: X11 AND BSD-3-Clause size: 799196 timestamp: 1686077139703 +- platform: linux-64 + name: networkx + version: 3.2.1 + category: main + manager: conda + dependencies: + - python >=3.9 + url: https://conda.anaconda.org/conda-forge/noarch/networkx-3.2.1-pyhd8ed1ab_0.conda + hash: + md5: 425fce3b531bed6ec3c74fab3e5f0a1c + sha256: 7629aa4f9f8cdff45ea7a4701fe58dccce5bf2faa01c26eb44cbb27b7e15ca9d + build: pyhd8ed1ab_0 + arch: x86_64 + subdir: linux-64 + build_number: 0 + constrains: + - matplotlib >=3.5 + - scipy >=1.9,!=1.11.0,!=1.11.1 + - numpy >=1.22 + - pandas >=1.4 + license: BSD-3-Clause + license_family: BSD + noarch: python + size: 1149552 + timestamp: 1698504905258 +- platform: osx-64 + name: networkx + version: 3.2.1 + category: main + manager: conda + dependencies: + - python >=3.9 + url: https://conda.anaconda.org/conda-forge/noarch/networkx-3.2.1-pyhd8ed1ab_0.conda + hash: + md5: 425fce3b531bed6ec3c74fab3e5f0a1c + sha256: 7629aa4f9f8cdff45ea7a4701fe58dccce5bf2faa01c26eb44cbb27b7e15ca9d + build: pyhd8ed1ab_0 + arch: x86_64 + subdir: osx-64 + build_number: 0 + constrains: + - matplotlib >=3.5 + - scipy >=1.9,!=1.11.0,!=1.11.1 + - numpy >=1.22 + - pandas >=1.4 + license: BSD-3-Clause + license_family: BSD + noarch: python + size: 1149552 + timestamp: 1698504905258 +- platform: osx-arm64 + name: networkx + version: 3.2.1 + category: main + manager: conda + dependencies: + - python >=3.9 + url: https://conda.anaconda.org/conda-forge/noarch/networkx-3.2.1-pyhd8ed1ab_0.conda + hash: + md5: 425fce3b531bed6ec3c74fab3e5f0a1c + sha256: 7629aa4f9f8cdff45ea7a4701fe58dccce5bf2faa01c26eb44cbb27b7e15ca9d + build: pyhd8ed1ab_0 + arch: aarch64 + subdir: osx-arm64 + build_number: 0 + constrains: + - matplotlib >=3.5 + - scipy >=1.9,!=1.11.0,!=1.11.1 + - numpy >=1.22 + - pandas >=1.4 + license: BSD-3-Clause + license_family: BSD + noarch: python + size: 1149552 + timestamp: 1698504905258 +- platform: win-64 + name: networkx + version: 3.2.1 + category: main + manager: conda + dependencies: + - python >=3.9 + url: https://conda.anaconda.org/conda-forge/noarch/networkx-3.2.1-pyhd8ed1ab_0.conda + hash: + md5: 425fce3b531bed6ec3c74fab3e5f0a1c + sha256: 7629aa4f9f8cdff45ea7a4701fe58dccce5bf2faa01c26eb44cbb27b7e15ca9d + build: pyhd8ed1ab_0 + arch: x86_64 + subdir: win-64 + build_number: 0 + constrains: + - matplotlib >=3.5 + - scipy >=1.9,!=1.11.0,!=1.11.1 + - numpy >=1.22 + - pandas >=1.4 + license: BSD-3-Clause + license_family: BSD + noarch: python + size: 1149552 + timestamp: 1698504905258 - platform: linux-64 name: numpy - version: 1.25.1 + version: 1.26.2 category: main manager: conda dependencies: + - libblas >=3.9.0,<4.0a0 - libcblas >=3.9.0,<4.0a0 + - libgcc-ng >=12 - liblapack >=3.9.0,<4.0a0 - libstdcxx-ng >=12 - python >=3.10,<3.11.0a0 - python_abi 3.10.* *_cp310 - - libblas >=3.9.0,<4.0a0 - - libgcc-ng >=12 - url: https://conda.anaconda.org/conda-forge/linux-64/numpy-1.25.1-py310ha4c1d20_0.conda + url: https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.2-py310hb13e2d6_0.conda hash: - md5: 3810cbf2635cb1d0edb97715d4ad74e7 - sha256: 38ec15fe0afe9fb90bd50314ccd506f0e7d1642db0c7eb2b77627d448aa9ee6c - build: py310ha4c1d20_0 + md5: d3147cfbf72d6ae7bba10562208f6def + sha256: f5ea7769beb7827f4f5858d28bbdbc814c01649cb8cb81cccbba476ebe3798cd + build: py310hb13e2d6_0 arch: x86_64 subdir: linux-64 build_number: 0 @@ -5285,25 +5385,26 @@ package: - numpy-base <0a0 license: BSD-3-Clause license_family: BSD - size: 6816069 - timestamp: 1688887559516 + size: 6922736 + timestamp: 1700875005993 - platform: osx-64 name: numpy - version: 1.25.1 + version: 1.26.2 category: main manager: conda dependencies: + - __osx >=10.9 + - libblas >=3.9.0,<4.0a0 - libcblas >=3.9.0,<4.0a0 - - libcxx >=15.0.7 + - libcxx >=16.0.6 - liblapack >=3.9.0,<4.0a0 - python >=3.10,<3.11.0a0 - python_abi 3.10.* *_cp310 - - libblas >=3.9.0,<4.0a0 - url: https://conda.anaconda.org/conda-forge/osx-64/numpy-1.25.1-py310h7451ae0_0.conda + url: https://conda.anaconda.org/conda-forge/osx-64/numpy-1.26.2-py310h2a7ecf2_0.conda hash: - md5: 525db32fd93b63f2f7ca3ece8576b9c8 - sha256: 32fe99f86e998169999514fb7f96695fdec9215bd0e7061425c1b4e399ca2cae - build: py310h7451ae0_0 + md5: 1ab91874430a8d4ba9b622082e08f438 + sha256: e607a924f5fce4b17262c4e091efcf6d1ccfded59212d9ac786ecf89f4cecae4 + build: py310h2a7ecf2_0 arch: x86_64 subdir: osx-64 build_number: 0 @@ -5311,25 +5412,27 @@ package: - numpy-base <0a0 license: BSD-3-Clause license_family: BSD - size: 6378643 - timestamp: 1688887678910 + size: 6368647 + timestamp: 1700875132575 - platform: osx-arm64 name: numpy - version: 1.25.1 + version: 1.26.2 category: main manager: conda dependencies: + - __osx >=10.9 + - libblas >=3.9.0,<4.0a0 - libcblas >=3.9.0,<4.0a0 - - libcxx >=15.0.7 + - libcxx >=16.0.6 - liblapack >=3.9.0,<4.0a0 + - python >=3.10,<3.11.0a0 - python >=3.10,<3.11.0a0 *_cpython - python_abi 3.10.* *_cp310 - - libblas >=3.9.0,<4.0a0 - url: https://conda.anaconda.org/conda-forge/osx-arm64/numpy-1.25.1-py310haa1e00c_0.conda + url: https://conda.anaconda.org/conda-forge/osx-arm64/numpy-1.26.2-py310h30ee222_0.conda hash: - md5: e6cb6d386238dd8cce9b403b8f91a33f - sha256: 80a838a20e053efe45da5bdad7b21383eda398384caae77453330386c705012a - build: py310haa1e00c_0 + md5: baea68ccbc288b1a8afb237ddee162c8 + sha256: d1c03544798b3916c97571dcc3ec3cde8ad2d9afe2f7ab82277fdadef4186e46 + build: py310h30ee222_0 arch: aarch64 subdir: osx-arm64 build_number: 0 @@ -5337,27 +5440,27 @@ package: - numpy-base <0a0 license: BSD-3-Clause license_family: BSD - size: 5634016 - timestamp: 1688887693258 + size: 5562504 + timestamp: 1700875196095 - platform: win-64 name: numpy - version: 1.25.1 + version: 1.26.2 category: main manager: conda dependencies: - - python >=3.10,<3.11.0a0 - - python_abi 3.10.* *_cp310 - libblas >=3.9.0,<4.0a0 - - vc >=14.2,<15 - - ucrt >=10.0.20348.0 - libcblas >=3.9.0,<4.0a0 - liblapack >=3.9.0,<4.0a0 + - python >=3.10,<3.11.0a0 + - python_abi 3.10.* *_cp310 + - ucrt >=10.0.20348.0 + - vc >=14.2,<15 - vc14_runtime >=14.29.30139 - url: https://conda.anaconda.org/conda-forge/win-64/numpy-1.25.1-py310hd02465a_0.conda + url: https://conda.anaconda.org/conda-forge/win-64/numpy-1.26.2-py310hf667824_0.conda hash: - md5: 922f75b8698c5b9909bf03c658898117 - sha256: 25e07d23dd78641537082fd9b0c4183784fcc1d84c65f207d6e2e7ede7702c8f - build: py310hd02465a_0 + md5: 4aa7d608c4c28f9f7927df760be4d52a + sha256: 574818695f0d320e0bc8405dd777f154e065deecfdc3546878443ce8dfe5af35 + build: py310hf667824_0 arch: x86_64 subdir: win-64 build_number: 0 @@ -5365,8 +5468,8 @@ package: - numpy-base <0a0 license: BSD-3-Clause license_family: BSD - size: 5985520 - timestamp: 1688887651966 + size: 6001822 + timestamp: 1700875316861 - platform: linux-64 name: openjpeg version: 2.5.0 @@ -6159,6 +6262,95 @@ package: license_family: BSD size: 6130 timestamp: 1669071917673 +- platform: linux-64 + name: pyyaml + version: 6.0.1 + category: main + manager: conda + dependencies: + - libgcc-ng >=12 + - python >=3.10,<3.11.0a0 + - python_abi 3.10.* *_cp310 + - yaml >=0.2.5,<0.3.0a0 + url: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.1-py310h2372a71_1.conda + hash: + md5: bb010e368de4940771368bc3dc4c63e7 + sha256: aa78ccddb0a75fa722f0f0eb3537c73ee1219c9dd46cea99d6b9eebfdd780f3d + build: py310h2372a71_1 + arch: x86_64 + subdir: linux-64 + build_number: 1 + license: MIT + license_family: MIT + size: 170627 + timestamp: 1695373587159 +- platform: osx-64 + name: pyyaml + version: 6.0.1 + category: main + manager: conda + dependencies: + - python >=3.10,<3.11.0a0 + - python_abi 3.10.* *_cp310 + - yaml >=0.2.5,<0.3.0a0 + url: https://conda.anaconda.org/conda-forge/osx-64/pyyaml-6.0.1-py310h6729b98_1.conda + hash: + md5: d964cec3e7972e44bc4a328134b9eaf1 + sha256: 00567f2cb2d1c8fede8fe7727f7bbd1c38cbca886814d612e162d5c936d8db1b + build: py310h6729b98_1 + arch: x86_64 + subdir: osx-64 + build_number: 1 + license: MIT + license_family: MIT + size: 160097 + timestamp: 1695373947773 +- platform: osx-arm64 + name: pyyaml + version: 6.0.1 + category: main + manager: conda + dependencies: + - python >=3.10,<3.11.0a0 + - python >=3.10,<3.11.0a0 *_cpython + - python_abi 3.10.* *_cp310 + - yaml >=0.2.5,<0.3.0a0 + url: https://conda.anaconda.org/conda-forge/osx-arm64/pyyaml-6.0.1-py310h2aa6e3c_1.conda + hash: + md5: 0e7ccdd121ce7b486f1de7917178387c + sha256: 7b8668cd86d2421c62ec241f840d84a600b854afc91383a509bbb60ba907aeec + build: py310h2aa6e3c_1 + arch: aarch64 + subdir: osx-arm64 + build_number: 1 + license: MIT + license_family: MIT + size: 158641 + timestamp: 1695373859696 +- platform: win-64 + name: pyyaml + version: 6.0.1 + category: main + manager: conda + dependencies: + - python >=3.10,<3.11.0a0 + - python_abi 3.10.* *_cp310 + - ucrt >=10.0.20348.0 + - vc >=14.2,<15 + - vc14_runtime >=14.29.30139 + - yaml >=0.2.5,<0.3.0a0 + url: https://conda.anaconda.org/conda-forge/win-64/pyyaml-6.0.1-py310h8d17308_1.conda + hash: + md5: ce279186f68d0f12812dc9955ea909a4 + sha256: ea51291e477b44c5bb9d91cc095db0dfe07b9576831e9682100d68c820c43ae3 + build: py310h8d17308_1 + arch: x86_64 + subdir: win-64 + build_number: 1 + license: MIT + license_family: MIT + size: 146195 + timestamp: 1695374085323 - platform: linux-64 name: rdma-core version: '28.9' @@ -7100,6 +7292,81 @@ package: license: LGPL-2.1 and GPL-2.0 size: 217804 timestamp: 1660346976440 +- platform: linux-64 + name: yaml + version: 0.2.5 + category: main + manager: conda + dependencies: + - libgcc-ng >=9.4.0 + url: https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2 + hash: + md5: 4cb3ad778ec2d5a7acbdf254eb1c42ae + sha256: a4e34c710eeb26945bdbdaba82d3d74f60a78f54a874ec10d373811a5d217535 + build: h7f98852_2 + arch: x86_64 + subdir: linux-64 + build_number: 2 + license: MIT + license_family: MIT + size: 89141 + timestamp: 1641346969816 +- platform: osx-64 + name: yaml + version: 0.2.5 + category: main + manager: conda + dependencies: [] + url: https://conda.anaconda.org/conda-forge/osx-64/yaml-0.2.5-h0d85af4_2.tar.bz2 + hash: + md5: d7e08fcf8259d742156188e8762b4d20 + sha256: 5301417e2c8dea45b401ffee8df3957d2447d4ce80c83c5ff151fc6bfe1c4148 + build: h0d85af4_2 + arch: x86_64 + subdir: osx-64 + build_number: 2 + license: MIT + license_family: MIT + size: 84237 + timestamp: 1641347062780 +- platform: osx-arm64 + name: yaml + version: 0.2.5 + category: main + manager: conda + dependencies: [] + url: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h3422bc3_2.tar.bz2 + hash: + md5: 4bb3f014845110883a3c5ee811fd84b4 + sha256: 93181a04ba8cfecfdfb162fc958436d868cc37db504c58078eab4c1a3e57fbb7 + build: h3422bc3_2 + arch: aarch64 + subdir: osx-arm64 + build_number: 2 + license: MIT + license_family: MIT + size: 88016 + timestamp: 1641347076660 +- platform: win-64 + name: yaml + version: 0.2.5 + category: main + manager: conda + dependencies: + - vc >=14.1,<15.0a0 + - vs2015_runtime >=14.16.27012 + url: https://conda.anaconda.org/conda-forge/win-64/yaml-0.2.5-h8ffe710_2.tar.bz2 + hash: + md5: adbfb9f45d1004a26763652246a33764 + sha256: 4e2246383003acbad9682c7c63178e2e715ad0eb84f03a8df1fbfba455dfedc5 + build: h8ffe710_2 + arch: x86_64 + subdir: win-64 + build_number: 2 + license: MIT + license_family: MIT + size: 63274 + timestamp: 1641347623319 - platform: linux-64 name: zstd version: 1.5.2 diff --git a/examples/rerun/pixi.toml b/examples/rerun_example/pixi.toml similarity index 64% rename from examples/rerun/pixi.toml rename to examples/rerun_example/pixi.toml index cdbaa1103..4f76c2daf 100644 --- a/examples/rerun/pixi.toml +++ b/examples/rerun_example/pixi.toml @@ -7,9 +7,11 @@ channels = ["conda-forge"] platforms = ["linux-64", "win-64", "osx-64", "osx-arm64"] [tasks] -start = "python dna_example.py" +start = "python force_driven_lockfile_graph.py" [dependencies] -rerun-sdk = "0.10.1.*" -numpy = "1.25.1.*" +numpy = ">=1.26.2,<1.27" python = "3.10.12.*" +rerun-sdk = ">=0.10.1,<0.11" +networkx = ">=3.2.1,<3.3" +pyyaml = ">=6.0.1,<6.1" From c8a8547aab035e27a453df407591a3bf740d756a Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 6 Dec 2023 23:21:55 +0100 Subject: [PATCH 2/2] misc: more tweaking --- .../force_driven_lockfile_graph.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/examples/rerun_example/force_driven_lockfile_graph.py b/examples/rerun_example/force_driven_lockfile_graph.py index 45315c2e3..a8225a8f2 100644 --- a/examples/rerun_example/force_driven_lockfile_graph.py +++ b/examples/rerun_example/force_driven_lockfile_graph.py @@ -25,12 +25,6 @@ rr.init("fdg", spawn=True) rr.connect() -# Force-Directed Simulation Parameters -iterations = 100 -repulsive_force = 0.04 -attractive_force = 0.002 - - def hash_string_to_int(string): return int(hashlib.sha256(string.encode('utf-8')).hexdigest(), 16) % (10 ** 8) @@ -49,8 +43,13 @@ def get_color_for_node(node): def apply_forces_and_log(graph, pos): damping = 0.9 - max_force = 10 - degree_scale = 0.9 # Scale factor for degree-based forces + max_force = 1 + degree_scale = 2 # Scale factor for degree-based forces + dist_scale = 0.5 + + iterations = 1000 + repulsive_force = 0.01 + attractive_force = 0.005 for iteration in range(iterations): force = {node: np.zeros(3) for node in graph} @@ -59,7 +58,7 @@ def apply_forces_and_log(graph, pos): for i, node1 in enumerate(graph): for node2 in list(graph)[i + 1:]: diff = pos[node1] - pos[node2] - dist = np.linalg.norm(diff) + 1e-9 # Avoid divide-by-zero + dist = (np.linalg.norm(diff) + 1e-9) * dist_scale degree_factor = ( (graph.degree(node1) + graph.degree(node2)) * degree_scale) repel = repulsive_force * degree_factor / dist ** 2 @@ -71,7 +70,7 @@ def apply_forces_and_log(graph, pos): for edge in graph.edges(): u, v = edge diff = pos[u] - pos[v] - dist = np.linalg.norm(diff) + dist = dist = (np.linalg.norm(diff) + 1e-9) * dist_scale if dist > 0: degree_factor = (graph.degree(u) + graph.degree(v)) * degree_scale attract = (attractive_force * dist ** 2) / degree_factor