Merge branch 'develop' into feature/updates

OpenBB-finance · Feb 9, 2023 · 3a9a126 · 3a9a126
2 parents 1672bf2 + 4b7cfd1
commit 3a9a126
Show file tree

Hide file tree

Showing 55 changed files with 19,271 additions and 5,194 deletions.
diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
@@ -23,7 +23,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
-          version: 1.1.13
+          version: 1.3.2
           virtualenvs-create: true
           virtualenvs-in-project: true
 
@@ -76,4 +76,3 @@ jobs:
           # You can swap them out with your own user credentials.
           user_name: github-actions[bot]
           user_email: 41898282+github-actions[bot]@users.noreply.github.com
-
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -31,7 +31,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1
       with:
-        version: 1.1.13
+        version: 1.3.2
         virtualenvs-create: true
         virtualenvs-in-project: true
 

diff --git a/.github/workflows/intel_macos_build.yml b/.github/workflows/intel_macos_build.yml
@@ -22,7 +22,7 @@ jobs:
     steps:
       # Checkout repository main branch. this allows for the commit hashes to line up
       - name: Checkout
-        uses: actions/checkout@v2.4.2
+        uses: actions/checkout@v3
       - name: Git Log
         run: git log
       # Install create-dmg
@@ -36,7 +36,7 @@ jobs:
           echo $PATH
       # Set up caching for conda env so that the workflow runs quickly after the first time
       - name: Setup Conda Caching
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/conda_pkgs_dir
           key: conda-macos-3-9-${{ hashFiles('build/conda/conda-3-9-env-full.yaml') }}
@@ -53,19 +53,19 @@ jobs:
           activate-environment: build_env
           use-only-tar-bz2: true # Needed for caching some reason
       - name: Run Poetry
-        run: poetry install
-      - name: Install Pyinstaller
-        run: poetry install -E installer
-      - name: Install Portfolio Optimization and Forecasting Toolkits
-        run: poetry install -E all
+        run: |
+          pip list
+          python -m pip install -r requirements-full.txt
       - name: Install Specific Papermill
         run: |
+          pip list
           pip uninstall papermill -y
           pip install git+https://github.com/nteract/papermill.git@main
       # This is basically the same as build4mac.sh
       - name: Build Bundle
         run: |
-          pyinstaller build/pyinstaller/terminal.spec --clean
+          pip list
+          python -m PyInstaller build/pyinstaller/terminal.spec --clean
           osascript build/pyinstaller/setup_icons.applescript
           mkdir DMG
           cp -r build/pyinstaller/macOS_package_assets/* DMG/
@@ -149,7 +149,7 @@ jobs:
           rm -rf build/terminal
           rm -rf dist
           rm -rf DMG
-      - name: Notorize DMG
+      - name: Notarize DMG
         env:
           NOTARIZE_APPLE_ID: ${{ secrets.NOTARIZE_APPLE_ID }}
           NOTARIZE_APPLE_PWD: ${{ secrets.NOTARIZE_APPLE_PWD }}
@@ -178,7 +178,7 @@ jobs:
       - name: Unmount DMG
         run: hdiutil detach /dev/disk2
       - name: Run Integration Tests
-        run: /Users/openbb/Desktop/OpenBB\ Terminal/.OpenBB/OpenBBTerminal /Users/openbb/actions-runner/_work/OpenBBTerminal/OpenBBTerminal/openbb_terminal/miscellaneous/integration_tests_scripts -t
+        run: /Users/openbb/Desktop/OpenBB\ Terminal/.OpenBB/OpenBBTerminal -t -s forecast cryptocurrency reports alternative economy futures econometrics dashboards portfolio stocks/test_stocks_options_screen.openbb stocks/test_stocks_options.openbb forex etf stocks/test_stocks_fa.openbb
       - name: Remove OpenBB Folder
         run: rm -rf /Users/openbb/Desktop/OpenBB\ Terminal
       - name: Remove OpenBB Exports Folder

diff --git a/.github/workflows/m1_macos_build.yml b/.github/workflows/m1_macos_build.yml
@@ -22,7 +22,7 @@ jobs:
     steps:
       # Checkout repository main branch. this allows for the commit hashes to line up
       - name: Checkout
-        uses: actions/checkout@v2.4.2
+        uses: actions/checkout@v3
       - name: Git Log
         run: git log
       # Install create-dmg
@@ -36,7 +36,7 @@ jobs:
           echo $PATH
       # Set up caching for conda env so that the workflow runs quickly after the first time
       - name: Setup Conda Caching
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/conda_pkgs_dir
           key: conda-macos-3-9-${{ hashFiles('build/conda/conda-3-9-env-full.yaml') }}
@@ -55,13 +55,9 @@ jobs:
           use-only-tar-bz2: true # Needed for caching some reason
 
       - name: Run Poetry
-        run: poetry install -vvv
-
-      - name: Install Pyinstaller
-        run: poetry install -E installer
-
-      - name: Poetry Install Portfolio Optimization and Forecasting Toolkits
-        run: poetry install -E all
+        run: |
+          pip list
+          python -m pip install -r requirements-full.txt
 
       - name: Install Specific Papermill
         run: |
@@ -71,13 +67,102 @@ jobs:
       # This is basically the same as build4mac.sh
       - name: Build Bundle
         run: |
-          pyinstaller build/pyinstaller/terminal.spec --clean
+          python -m PyInstaller build/pyinstaller/terminal.spec --clean
           osascript build/pyinstaller/setup_icons.applescript
           mkdir DMG
           cp -r build/pyinstaller/macOS_package_assets/* DMG/
           mv dist/OpenBBTerminal DMG/OpenBB\ Terminal/.OpenBB
+      - name: Creating Keychain
+        env:
+          MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }}
+          MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
+          MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }}
+          MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
+        run: | # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity
+          echo "Ensuring Keychain with same name does not exist"
+          rm -rf /Users/openbb/Library/Keychains/build.keychain-db
+          echo "Decoding certificate"
+          echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
+          echo "Creating Keychain"
+          security create-keychain -p $MACOS_KEYCHAIN_PWD build.keychain
+          echo "Setting Default Keychain"
+          security default-keychain -s build.keychain
+          echo "Unlocking Keychain"
+          security unlock-keychain -p $MACOS_KEYCHAIN_PWD build.keychain
+          echo "Importing Keychain"
+          security import certificate.p12 -k build.keychain -P $MACOS_CERTIFICATE_PWD -T /usr/bin/codesign
+          echo "Setting Partition List"
+          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $MACOS_KEYCHAIN_PWD build.keychain
+      - name: Codesign Specific Binaries
+        env:
+          MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }}
+          MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
+          MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }}
+          MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
+        run: |
+          echo "Code Signing SO Files"
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/*.so
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/*/*.so
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/*/*/*.so
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/*/*/*/*.so
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/*/*/*/*/*.so
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/*/*/*/*/*/*.so
+          echo "Code Signing DYLIB Files"
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/*.dylib
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/*/*.dylib
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/*/*/*.dylib
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/*/*/*/*/*.dylib
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/prophet/.dylibs/*.dylib
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/prophet/stan_model/cmdstan-2.26.1/stan/lib/stan_math/lib/tbb/*.dylib
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/torch/.dylibs/*.dylib
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/sklearn/.dylibs/*.dylib
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/scipy/.dylibs/*.dylib
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/xgboost/.dylibs/*.dylib
+
+          echo "Code Signing Other Files"
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/prophet/stan_model/prophet_model.bin
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/prophet/stan_model/cmdstan-2.26.1/bin/diagnose
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/prophet/stan_model/cmdstan-2.26.1/bin/print
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/prophet/stan_model/cmdstan-2.26.1/bin/stanc
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/prophet/stan_model/cmdstan-2.26.1/bin/stansummary
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/pyarrow/plasma-store-server
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/torch/bin/protoc
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/torch/bin/protoc-3.13.0.0
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/torch/bin/torch_shm_manager
+          echo "Code Sign OpenBB Executable File"
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY DMG/OpenBB\ Terminal/.OpenBB/OpenBBTerminal
       - name: Create DMG
         run: create-dmg --volname "OpenBB Terminal" --volicon "images/dmg_volume.icns" --background "images/openbb_dmg_background.png" --icon "OpenBB Terminal" 190 250 --window-pos 190 120 --window-size 800 400 --icon-size 100 --text-size 14 --app-drop-link 600 250 --eula LICENSE --format UDZO --no-internet-enable "OpenBB Terminal".dmg DMG
+      - name: Code Sign DMG Installer
+        env:
+          MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }}
+          MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
+          MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }}
+          MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
+        run: |
+          echo "Code Signing DMG Installer"
+          /usr/bin/codesign --deep --force --verify --verbose --options runtime --entitlements "build/pyinstaller/entitlements.plist" -s $MACOS_CODESIGN_IDENTITY OpenBB\ Terminal.dmg
+          echo "Verify Signing"
+          codesign --verify --deep --strict --verbose=2 OpenBB\ Terminal.dmg
+      - name: Deleting Previous Keychain
+        run: |
+          echo "Deleting Previous Keychain to Clean Instance"
+          rm -rf /Users/openbb/Library/Keychains/build.keychain-db
+      - name: Clean up Build Artifacts
+        run: |
+          rm -rf build/terminal
+          rm -rf dist
+          rm -rf DMG
+      - name: Notorize DMG
+        env:
+          NOTARIZE_APPLE_ID: ${{ secrets.NOTARIZE_APPLE_ID }}
+          NOTARIZE_APPLE_PWD: ${{ secrets.NOTARIZE_APPLE_PWD }}
+          NOTARIZE_APPLE_TEAM_ID: ${{ secrets.NOTARIZE_APPLE_TEAM_ID }}
+        run: |
+          xcrun notarytool submit OpenBB\ Terminal.dmg --apple-id "$NOTARIZE_APPLE_ID" --password "$NOTARIZE_APPLE_PWD" --team-id "$NOTARIZE_APPLE_TEAM_ID" --wait
+      - name: Staple
+        run: |
+          xcrun stapler staple "OpenBB Terminal.dmg"
       - name: Clean up Build Artifacts
         run: |
           rm -rf build/terminal
@@ -102,7 +187,7 @@ jobs:
       - name: Unmount DMG
         run: hdiutil detach /dev/disk4
       - name: Run Integration Tests
-        run: /Users/openbb/Desktop/OpenBB\ Terminal/.OpenBB/OpenBBTerminal /Users/openbb/actions-runner/_work/OpenBBTerminal/OpenBBTerminal/openbb_terminal/miscellaneous/integration_tests_scripts -t
+        run: /Users/openbb/Desktop/OpenBB\ Terminal/.OpenBB/OpenBBTerminal -t -s forecast cryptocurrency reports alternative economy futures econometrics dashboards portfolio stocks/test_stocks_options_screen.openbb stocks/test_stocks_options.openbb forex etf stocks/test_stocks_fa.openbb
       - name: Remove OpenBB Folder
         run: rm -rf /Users/openbb/Desktop/OpenBB\ Terminal
       - name: Remove OpenBB Exports Folder

diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml
@@ -13,7 +13,7 @@ on:
   schedule:
     - cron: "0 0 * * *"
 
-permissions: 
+permissions:
   actions: write
 
 jobs:
@@ -26,7 +26,7 @@ jobs:
           workflow: windows10_build.yml
           token: ${{ secrets.GITHUB_TOKEN }}
           wait-for-completion-timeout: 2h
-  
+
   trigger-macos-build:
     runs-on: ubuntu-latest
     steps:
@@ -36,7 +36,7 @@ jobs:
           workflow: m1_macos_build.yml
           token: ${{ secrets.GITHUB_TOKEN }}
           wait-for-completion-timeout: 2h
-  
+
   trigger-intel-build:
     runs-on: ubuntu-latest
     steps:
@@ -46,7 +46,7 @@ jobs:
           workflow: intel_macos_build.yml
           token: ${{ secrets.GITHUB_TOKEN }}
           wait-for-completion-timeout: 2h
-  
+
   trigger-docker-build:
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -56,7 +56,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
-          version: 1.1.13
+          version: 1.3.2
           virtualenvs-create: true
           virtualenvs-in-project: true
 

diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
@@ -77,7 +77,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
-          version: 1.1.13
+          version: 1.3.2
           virtualenvs-create: true
           virtualenvs-in-project: true
 

diff --git a/.github/workflows/windows10_build.yml b/.github/workflows/windows10_build.yml
@@ -17,7 +17,7 @@ concurrency:
 jobs:
   Windows-Build:
     name: Windows10 Build ENV
-    runs-on: windows-latest
+    runs-on: windows-2019
     # Configuring -------------
     steps:
       - name: Setup Windows Git Configuration # This is under the assumption that git is already installed on the system
@@ -26,7 +26,7 @@ jobs:
           git config --system core.longpaths true
 
       - name: Checkout
-        uses: actions/checkout@v2.4.2
+        uses: actions/checkout@v3
 
       - name: Set up Python & Create OpenBB Environment
         uses: actions/setup-python@v4
@@ -38,10 +38,9 @@ jobs:
           python -m venv venv
           source venv/Scripts/activate
           python -m pip install --upgrade pip
-          python -m pip install poetry==1.1.13
           python -m pip install setuptools==64.0.2 wheel
           python -m pip install charset_normalizer==2.1.1
-          python -m poetry install -E installer -E all
+          python -m pip install -r requirements-full.txt
           python -m pip uninstall papermill -y
           python -m pip install git+https://github.com/nteract/papermill.git@main
           python -m PyInstaller build/pyinstaller/terminal.spec --clean
@@ -67,7 +66,7 @@ jobs:
           git config --system core.longpaths true
 
       - name: Checkout
-        uses: actions/checkout@v2.4.2
+        uses: actions/checkout@v3
 
       - name: Download Artifact
         uses: actions/download-artifact@v2
@@ -92,7 +91,7 @@ jobs:
 
       - name: Run Integration Tests
         run: |
-          dist\OpenBBTerminal\OpenBBTerminal.exe -t
+          dist\OpenBBTerminal\OpenBBTerminal.exe -t -s forecast cryptocurrency reports alternative economy futures econometrics dashboards portfolio stocks/test_stocks_options_screen.openbb stocks/test_stocks_options.openbb forex etf stocks/test_stocks_fa.openbb
 
       # Cleaning ------------------------
       # Make sure to add Remove-Item C:\Users\Administrator\Desktop\OPENBB-exports -Recurse whenever integration tests get fixed

diff --git a/build/conda/conda-3-8-env-full.yaml b/build/conda/conda-3-8-env-full.yaml
@@ -4,5 +4,6 @@ channels:
 dependencies:
   - python=3.8
   - pip
-  - poetry=1.1.13
-  - lightgbm=3.3.3
+  - conda-forge::lightgbm=3.3.3
+  - pip:
+    - poetry==1.3.2
diff --git a/build/conda/conda-3-8-env.yaml b/build/conda/conda-3-8-env.yaml
@@ -4,4 +4,5 @@ channels:
 dependencies:
   - python=3.8
   - pip
-  - poetry=1.1.13
+  - pip:
+    - poetry==1.3.2
diff --git a/build/conda/conda-3-9-env-full.yaml b/build/conda/conda-3-9-env-full.yaml
@@ -2,7 +2,8 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.9.6
+  - python=3.9.13
   - pip
-  - poetry=1.1.13
-  - lightgbm=3.3.5
+  - conda-forge::lightgbm=3.3.3
+  - pip:
+    - poetry==1.3.2
diff --git a/build/conda/conda-3-9-env.yaml b/build/conda/conda-3-9-env.yaml
@@ -2,6 +2,7 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.9
+  - python=3.9.13
   - pip
-  - poetry=1.1.13
+  - pip:
+    - poetry==1.3.2
diff --git a/build/conda/conda-3-9-windows.yaml b/build/conda/conda-3-9-windows.yaml
@@ -0,0 +1,8 @@
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.9
+  - poetry==1.3.2
+  - setuptools=64.0.2
+  - pip