Code Cleanup (#23)

* code cleanup copy of arangoml/dgl-adapter@6b36a50 * attempt fix: `pip install torch` * attempt fix: remove `-e` * attempt fix: remove cache * fix: bad comment * attempt fix: `pip install torch-sparse` * attempt fix: torch installations * attempt fix: upgrade pip hitting a wall... * cleanup: workflows * cleanup: `adapter.py` * cleanup: workflows * fix: install publishing dependencies * extracting more logic into separate functions * fix: mypy * rename kwargs * bump `adbpyg-adapter` to `1.1.2` in notebooks * fix param order * bump * Delete build_self_hosted.yml * cleanup workflows * Update build.yml * Update build.yml * Update build.yml * parameter renaming, new `use_async` param * fix abc * fix mypy * add `strict` param to other adb -> pyg methods * Update setup.py
arangoml · Oct 25, 2023 · bf70b9f · bf70b9f
1 parent 723579b
commit bf70b9f
Show file tree

Hide file tree

Showing 12 changed files with 2,517 additions and 2,264 deletions.
diff --git a/.github/workflows/analyze.yml b/.github/workflows/analyze.yml
@@ -37,7 +37,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,9 +1,8 @@
 name: build
 on:
   workflow_dispatch:
-  push:
-    branches: [ master ]
   pull_request:
+  push:
     branches: [ master ]
 env:
   PACKAGE_DIR: adbpyg_adapter
@@ -16,31 +15,44 @@ jobs:
         python: ["3.8", "3.9", "3.10", "3.11"]
     name: Python ${{ matrix.python }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
       - name: Setup Python ${{ matrix.python }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
           cache: 'pip'
           cache-dependency-path: setup.py
+
+      - name: Set up ArangoDB Instance via Docker
+        run: docker create --name adb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
+
+      - name: Start ArangoDB Instance
+        run: docker start adb
+
       - name: Install packages
-        run: pip install torch && pip install -e .[dev]
+        run: |
+          pip install torch
+          pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
+          pip install -e .[dev]
+
       - name: Run black
         run: black --check --verbose --diff --color ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
+
       - name: Run flake8
         run: flake8 ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
+
       - name: Run isort
         run: isort --check --profile=black ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
+
       - name: Run mypy
         run: mypy ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
-      - name: Set up ArangoDB Instance via Docker
-        run: docker create --name adb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
-      - name: Start ArangoDB Instance
-        run: docker start adb
+
       - name: Run pytest
         run: pytest --cov=${{env.PACKAGE_DIR}} --cov-report xml --cov-report term-missing -v --color=yes --no-cov-on-fail --code-highlight=yes
+
       - name: Publish to coveralls.io
-        if: matrix.python == '3.8'
+        if: matrix.python == '3.10'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: coveralls --service=github
diff --git a/.github/workflows/build_self_hosted.yml b/.github/workflows/build_self_hosted.yml
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -3,70 +3,23 @@ on:
   workflow_dispatch:
   release:
     types: [published]
-env:
-  PACKAGE_DIR: adbpyg_adapter
-  TESTS_DIR: tests
 jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python: ["3.8", "3.9", "3.10", "3.11"]
-    name: Python ${{ matrix.python }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python }}
-          cache: 'pip'
-          cache-dependency-path: setup.py
-      - name: Install packages
-        run: pip install torch && pip install -e .[dev]
-      - name: Run black
-        run: black --check --verbose --diff --color ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
-      - name: Run flake8
-        run: flake8 ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
-      - name: Run isort
-        run: isort --check --profile=black ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
-      - name: Run mypy
-        run: mypy ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
-      - name: Set up ArangoDB Instance via Docker
-        run: docker create --name adb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
-      - name: Start ArangoDB Instance
-        run: docker start adb
-      - name: Run pytest
-        run: pytest --cov=${{env.PACKAGE_DIR}} --cov-report xml --cov-report term-missing -v --color=yes --no-cov-on-fail --code-highlight=yes
-      - name: Publish to coveralls.io
-        if: matrix.python == '3.8'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: coveralls --service=github
-
   release:
-    needs: build
     runs-on: ubuntu-latest
     name: Release package
     steps:
-      - uses: actions/checkout@v3
-      - name: Setup Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.8"
-          cache: 'pip'
-          cache-dependency-path: setup.py
+      - uses: actions/checkout@v4
 
       - name: Fetch complete history for all tags and branches
         run: git fetch --prune --unshallow
 
-      - name: Install packages
-        run: |
-          pip install setuptools wheel twine setuptools-scm[toml]
-          pip install torch
-          pip install .[dev]
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
 
-      - name: Remove (old) distribution
-        run: rm -rf dist
+      - name: Install release packages
+        run: pip install setuptools wheel twine setuptools-scm[toml]
 
       - name: Build distribution
         run: python setup.py sdist bdist_wheel
@@ -88,7 +41,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Update Changelog
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -100,12 +53,10 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Setup Python 3.8
+      - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
-          cache: 'pip'
-          cache-dependency-path: setup.py
+          python-version: "3.10"
 
       - name: Install release packages
         run: pip install wheel gitchangelog pystache

diff --git a/adbpyg_adapter/abc.py b/adbpyg_adapter/abc.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from abc import ABC
-from typing import Any, List, Set, Union
+from typing import Any, Set, Union
 
 from arango.graph import Graph as ArangoDBGraph
 from torch_geometric.data import Data, HeteroData
@@ -16,17 +16,17 @@ def __init__(self) -> None:
         raise NotImplementedError  # pragma: no cover
 
     def arangodb_to_pyg(
-        self, name: str, metagraph: ADBMetagraph, **query_options: Any
+        self, name: str, metagraph: ADBMetagraph, **adb_export_kwargs: Any
     ) -> Union[Data, HeteroData]:
         raise NotImplementedError  # pragma: no cover
 
     def arangodb_collections_to_pyg(
-        self, name: str, v_cols: Set[str], e_cols: Set[str], **query_options: Any
+        self, name: str, v_cols: Set[str], e_cols: Set[str], **adb_export_kwargs: Any
     ) -> Union[Data, HeteroData]:
         raise NotImplementedError  # pragma: no cover
 
     def arangodb_graph_to_pyg(
-        self, name: str, **query_options: Any
+        self, name: str, **adb_export_kwargs: Any
     ) -> Union[Data, HeteroData]:
         raise NotImplementedError  # pragma: no cover
 
@@ -37,30 +37,10 @@ def pyg_to_arangodb(
         metagraph: PyGMetagraph = {},
         explicit_metagraph: bool = True,
         overwrite_graph: bool = False,
-        **import_options: Any,
+        **adb_import_kwargs: Any,
     ) -> ArangoDBGraph:
         raise NotImplementedError  # pragma: no cover
 
-    def etypes_to_edefinitions(self, edge_types: List[EdgeType]) -> List[Json]:
-        raise NotImplementedError  # pragma: no cover
-
-    def ntypes_to_ocollections(
-        self, node_types: List[str], edge_types: List[EdgeType]
-    ) -> List[str]:
-        raise NotImplementedError  # pragma: no cover
-
-    def __fetch_adb_docs(self) -> None:
-        raise NotImplementedError  # pragma: no cover
-
-    def __insert_adb_docs(self) -> None:
-        raise NotImplementedError  # pragma: no cover
-
-    def __build_tensor_from_dataframe(self) -> None:
-        raise NotImplementedError  # pragma: no cover
-
-    def __build_dataframe_from_tensor(self) -> None:
-        raise NotImplementedError  # pragma: no cover
-
 
 class Abstract_ADBPyG_Controller(ABC):
     def _prepare_pyg_node(self, pyg_node: Json, node_type: str) -> Json: