Low performance hotspots

.azure_pipelines/README.md

@@ -0,0 +1,27 @@
+# Pipelines
+
+Note that the pipeline definitions in this folder are for azure pipelines, not
+github pipelines. There are differences between these systems.
+
+- `ci-eval-pr.yaml` is run for all PRs to _TruLens-Eval_. Success is needed for
+  merging into `main`.
+- `ci-eval.yaml` for _TruLens-Eval_ releases. This includes database migration
+  tests as well as running notebooks. Success is needed for merging into
+  `releases/*`. Also, any branch named `releases/*` needs to pass the pipeline
+  tests before merging into `main`.
+- `ci.yaml` is run for all PRs to _TruLens-Explain_. Success is needed for
+  merging into `main`.
+
+## More information
+
+- Branch protection rules. These specify what pipelines must succeed before a PR
+  can be merged. These are configured from the [Branches
+  settings](https://github.com/truera/trulens/settings/branches) panel.
+
+- Pipelines. Pipelines are described by the various `.yaml` files in this folder
+  and pointed to by the [Azure
+  Pipelines](https://dev.azure.com/truera/trulens/_build) definitions.
+
+- [Triggers
+  documentation](https://learn.microsoft.com/en-us/azure/devops/pipelines/build/triggers?view=azure-devops)
+  describes how to setup trigers (when a pipeline needs to run).

.azure_pipelines/ci-docs-pr.yaml

@@ -0,0 +1,35 @@
+# This a definition for azure pipelines, not github pipelines. There are
+# differences between these systems.
+
+trigger: none
+
+pr:
+  # Run this pipeline for any PR that wishes to push to main that changes code
+  # inside docs folder.
+  branches:
+    include:
+      - main
+  paths:
+    include:
+      - docs
+
+jobs:
+  - job: DocsCheck
+    displayName: "Check Documentation"
+    pool:
+      vmImage: "ubuntu-latest"
+    timeoutInMinutes: 30
+
+    strategy:
+      matrix:
+        latest:
+          python-version: "3.12"
+
+    steps:
+      - checkout: self
+        clean: true
+
+      - bash: |
+          echo "There are no doc checks at the moment."
+
+        displayName: "Dummy check"

.azure_pipelines/ci-eval-pr.yaml

@@ -1,52 +1,154 @@
+# This a definition for azure pipelines, not github pipelines. There are
+# differences between these systems.
+
+trigger: none
+
 pr:
+  # Run this pipeline for any PR that wishes to push to main that changes code
+  # inside trulens_eval folder.
   branches:
     include:
       - main
-      - releases/*
   paths:
     include:
       - trulens_eval
-      - docs/trulens_eval/*.ipynb
-
 
 jobs:
-  - job: PRBranchProtect
+  - job: PRBranchProtect # name seems to be important but cannot figure out why
     pool:
-      vmImage: "ubuntu-latest"
+      vmImage: ubuntu-latest
     timeoutInMinutes: 30
 
     strategy:
       matrix:
-        default:
-          condaEnvFileSuffix: "trulens-llm"
+        default: # name is important
+          python-version: 3.12
+          tests-folder: tests/unit
+        py308-static:
+          python-version: 3.8
+          tests-folder: tests/unit/static
+        py309-static:
+          python-version: 3.9
+          tests-folder: tests/unit/static
+        py310-static:
+          python-version: 3.10
+          tests-folder: tests/unit/static
+        py311-static:
+          python-version: 3.11
+          tests-folder: tests/unit/static
 
     steps:
       - checkout: self
         clean: true
-      - bash: echo "##vso[task.prependpath]$CONDA/bin"
-        displayName: Add conda to PATH
+
       - bash: |
+          echo "##vso[task.prependpath]$CONDA/bin"
+
           set -e
-          conda create  -y --quiet -n $(condaEnvFileSuffix) python=3
+          conda create -y --quiet \
+            -n $(python-version) \
+            python=$(python-version)
+
           conda init bash
-        displayName: Create Anaconda environment
-      - bash: |
-          set -e
-          source activate $(condaEnvFileSuffix)
-          which python
-        displayName: Which Python
+
+        displayName: Setup conda
+
       - bash: |
           set -e
-          source activate $(condaEnvFileSuffix)
-          pip install yapf==0.32.0
-          ./format.sh
-        displayName: Format Code
-      - bash: |
-          num_changed_files=`git ls-files --others -m --exclude-standard | wc -l`
+          source activate $(python-version)
+
+          pip install yapf==0.32.0 --verbose
+          pip install isort==5.10.1 --verbose
+
+          yapf --version
+          isort --vn
+
+          ./format.sh --eval
+
+          num_changed_files=`git ls-files --others -m --exclude-standard ./trulens_eval | wc -l`
           if [ $num_changed_files -ne 0 ]; then
             echo "The following files have changed after running format.sh. Please format your code and update the PR."
-            git ls-files --others -m --exclude-standard
+            git ls-files --others -m --exclude-standard ./trulens_eval
             echo "\n\nOutputting git diff for checked in files.\n"
             git diff
           fi
-        displayName: NoDiffCheck
+        condition: eq(variables['python-version'], 3.11) # only run format check with latest python version
+        displayName: Format diffs
+
+      - bash: |
+          set -e
+          source activate $(python-version)
+
+          cd ./trulens_eval
+          pip install -e . --verbose
+
+        displayName: Install trulens
+
+      - bash: |
+          set -e
+          source activate $(python-version)
+
+          pip install --verbose \
+            pytest==7.0.1 pytest-subtests pytest-azurepipelines
+
+        displayName: Install testing packages
+
+      - bash: |
+          set -e
+          source activate $(python-version)
+
+          python --version
+          pip --version
+          pip list --verbose
+
+        displayName: Describe python env
+
+      - bash: |
+          set -e
+          source activate $(python-version)
+
+          cd ./trulens_eval
+          python -m pytest --test-run-title="Required $(python-version) unit tests" $(tests-folder)
+
+        displayName: Unit tests with required packages
+
+      - bash: |
+          set -e
+          source activate $(python-version)
+
+          cd ./trulens_eval
+
+          # Once we start running notebooks in these tests, might have to do this again:
+          # pip install ipykernel
+          # python -m ipykernel install --user \
+          #  --name $(python-version) \
+          #  --display-name $(python-version)
+
+          pip install --verbose -r trulens_eval/requirements.optional.txt
+
+          python --version
+          pip --version
+          pip list --verbose
+
+        displayName: Install optional packages
+
+      - bash: |
+          set -e
+          source activate $(python-version)
+
+          cd ./trulens_eval
+
+          python -m pytest  --test-run-title="Optional $(python-version) unit tests" $(tests-folder)
+        env:
+          # enables optional tests, see tests/unit/test.py
+          TEST_OPTIONAL: true
+
+          # tests make use of various APIs:
+          OPENAI_API_KEY: $(OPENAI_API_KEY)
+          HUGGINGFACE_API_KEY: $(HUGGINGFACE_API_KEY)
+          PINECONE_API_KEY: $(PINECONE_API_KEY)
+          PINECONE_ENV: $(PINECONE_ENV)
+          HUGGINGFACEHUB_API_TOKEN: $(HUGGINGFACEHUB_API_TOKEN)
+
+        displayName: Unit tests with optional packages
+

.azure_pipelines/ci-eval.yaml

@@ -1,13 +1,8 @@
-trigger:
-  branches:
-    include:
-      - main
-  paths:
-    include:
-      - trulens_eval
-      - docs/trulens_eval/*.ipynb
+trigger: none
 
 pr:
+  # Run this pipeline for any PR from a branch that starts with "releases/" that
+  # changed code in trulens_eval folder or to eval notebooks in the docs folder.
   branches:
     include:
       - releases/*
@@ -16,76 +11,118 @@ pr:
       - trulens_eval
       - docs/trulens_eval/*.ipynb
 
+# schedules:
+#  - cron: "0 0 * * *"
+#    displayName: Daily midnight build
+#    branches:
+#      include:
+#        - main
 
 jobs:
-  - job: quickstartTests
+  - job: ReleaseTests
     pool:
       vmImage: "ubuntu-latest"
     timeoutInMinutes: 30
 
     strategy:
       matrix:
-        docs_notebooks:
+        latest:
+          pythonVersion: "3.11"
           condaEnvFileSuffix: "trulens-llm"
-          requirementsFile: "tests/docs_notebooks/requirements.txt"
-          testSubdirectory: "tests/docs_notebooks"
+
 
     steps:
       - checkout: self
         clean: true
+
       - bash: echo "##vso[task.prependpath]$CONDA/bin"
         displayName: Add conda to PATH
+
       - bash: |
           set -e
-          conda create  -y --quiet -n $(condaEnvFileSuffix) python=3
+          conda create  -y --quiet -n $(condaEnvFileSuffix) python=3.11
           conda init bash
         displayName: Create Anaconda environment
+
       - bash: |
           set -e
           source activate $(condaEnvFileSuffix)
           which python
+          python --version
         displayName: Which Python
+
       - bash: |
           set -e
           source activate $(condaEnvFileSuffix)
           cd ./trulens_eval
           pip install -e .
-          pip install -r $(requirementsFile)
+          pip install -r tests/docs_notebooks/requirements.txt
           pip install yapf==0.32.0
+          pip install isort==5.10.1
           pip install ipykernel
           python -m ipykernel install --user --name $(condaEnvFileSuffix) --display-name $(condaEnvFileSuffix)
         displayName: Install pip package
+
       - bash: |
           set -e
           source activate $(condaEnvFileSuffix)
-          ./format.sh
+          echo Using $(yapf --version) isort $(isort --vn)
+          ./format.sh --eval
         displayName: Format Code
+
       - bash: |
-          num_changed_files=`git ls-files --others -m --exclude-standard | wc -l`
+          num_changed_files=`git ls-files --others -m --exclude-standard ./trulens_eval | wc -l`
           if [ $num_changed_files -ne 0 ]; then
             echo "The following files have changed after running format.sh. Please format your code and update the PR."
-            git ls-files --others -m --exclude-standard
+            git ls-files --others -m --exclude-standard ./trulens_eval
             echo "\n\nOutputting git diff for checked in files.\n"
             git diff
           fi
         displayName: NoDiffCheck
+
       - bash: |
           source activate $(condaEnvFileSuffix)
           cd ./docs/trulens_eval
           pip install jupyter
           pip install nbmerge
           ../../.github/workflows/combine_nb_to_docs_testing.sh
         displayName: Run combine docs for testing
+
       - bash: |
           source activate $(condaEnvFileSuffix)
-          pip install pytest==7.0.1 pytest-azurepipelines langchain>=0.0.170 llama_index>=0.6.24 
-          pip install docarray hnswlib
-          echo '::group::piplist'
+
+          cd ./trulens_eval
+          pip install pytest==7.0.1 pytest-azurepipelines
+          pip install -r trulens_eval/requirements.optional.txt
+        displayName: Install optional deps
+
+      - bash: |
+          source activate $(condaEnvFileSuffix)
+          echo "$(python --version)"
           echo "$(pip list)"
-          echo '::endgroup::'
+        displayName: Pip list
+
+      - bash: |
+          source activate $(condaEnvFileSuffix)
+          cd ./trulens_eval
+
+          python -m pytest tests/e2e
+
+        displayName: Run e2e tests
+        env:
+          OPENAI_API_KEY: $(OPENAI_API_KEY)
+          HUGGINGFACE_API_KEY: $(HUGGINGFACE_API_KEY)
+          PINECONE_API_KEY: $(PINECONE_API_KEY)
+          PINECONE_ENV: $(PINECONE_ENV)
+          HUGGINGFACEHUB_API_TOKEN: $(HUGGINGFACEHUB_API_TOKEN)
+
+      - bash: |
+          source activate $(condaEnvFileSuffix)
           cd ./trulens_eval
-          python -m pytest $(testSubdirectory)
-        displayName: Run unit tests
+
+          python -m pytest tests/docs_notebooks
+
+        displayName: Run notebook tests
         env:
           OPENAI_API_KEY: $(OPENAI_API_KEY)
           HUGGINGFACE_API_KEY: $(HUGGINGFACE_API_KEY)