Upload 1140 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+AI/docs/docs/llms/llm-IntentClassifier-docs.jpg filter=lfs diff=lfs merge=lfs -text

AI/.coveragerc

@@ -0,0 +1,12 @@
+[report]
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    raise NotImplementedError
+    if __name__ == .__main__.:
+    def create_argument_parser
+    if typing.TYPE_CHECKING
+    # Ellipsis are used as placeholders in python 3 that will be overridden
+    \.\.\.
+[run]
+source = rasa

AI/.deepsource.toml

@@ -0,0 +1,12 @@
+version = 1
+
+test_patterns = ["tests/**"]
+
+exclude_patterns = ["docs/**"]
+
+[[analyzers]]
+name = "python"
+enabled = true
+
+  [analyzers.meta]
+  runtime_version = "3.x.x"

AI/.devcontainer/Dockerfile

@@ -0,0 +1,27 @@
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
+ARG VARIANT=3-bullseye
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+
+ENV PYTHONFAULTHANDLER=1 \
+  PYTHONUNBUFFERED=1 \
+  PYTHONHASHSEED=random \
+  PIP_NO_CACHE_DIR=off \
+  PIP_DISABLE_PIP_VERSION_CHECK=on \
+  PIP_DEFAULT_TIMEOUT=100
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "nne" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# [Optional] If your requirements rarely change, uncomment this section to add them to the image.
+# COPY requirements.txt /tmp/pip-tmp/
+# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+#    && rm -rf /tmp/pip-tmp
+RUN pip install poetry==1.1.10 pre-commit
+COPY ../poetry.lock ../pyproject.toml /tmp/pip-tmp/rasa/
+RUN cd /tmp/pip-tmp/rasa && poetry config virtualenvs.create false \
+  && poetry install --no-interaction --no-ansi --no-root
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>

AI/.devcontainer/devcontacener.json

@@ -0,0 +1,50 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3-postgres
+// Update the VARIANT arg in docker-compose.yml to pick a Python version
+{
+    "name": "Rasa Open Source",
+    "dockerComposeFile": "docker-compose.yml",
+    "service": "app",
+    "workspaceFolder": "/workspaces/rasa",
+    // Set *default* container specific settings.json values on container create.
+    "settings": {
+        "python.defaultInterpreterPath": "/usr/local/bin/python",
+        "python.linting.enabled": true,
+        "python.linting.pylintEnabled": true,
+        "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+        "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+        "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+        "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+        "python.linting.ruffPath": "/usr/local/py-utils/bin/ruff",
+        "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+        "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+        "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+        "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
+        "python.testing.pytestPath": "/usr/local/py-utils/bin/pytest"
+    },
+    // Add the IDs of extensions you want installed when the container is created.
+    "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance"
+    ],
+    // memory is required for frontend build...fails for machines with less than 10g
+    // "hostRequirements": {
+    //    "memory": "12gb"
+    // },
+    // Use 'forwardPorts' to make a list of ports inside the container available locally.
+    // This can be used to network with other containers or the host.
+    "forwardPorts": [
+        5005
+    ],
+    // Use 'postCreateCommand' to run commands after the container is created.
+    "updateContentCommand": "make install && make install-docs && cd / && mkdir example && rasa init --no-prompt --init-dir example",
+    // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+    //"remoteUser": "vscode",
+    "features": {
+        "docker-in-docker": "20.10",
+        "docker-from-docker": "20.10",
+        "git": "os-provided",
+        "github-cli": "latest",
+        "sshd": "latest"
+    }
+}

AI/.devcontainer/docker-compose.yml

@@ -0,0 +1,57 @@
+version: '3.8'
+
+services:
+  app:
+    build:
+      context: ..
+      dockerfile: .devcontainer/Dockerfile
+      args:
+        # Update 'VARIANT' to pick a version of Python: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+        # Append -bullseye or -buster to pin to an OS version.
+        # Use -bullseye variants on local arm64/Apple Silicon.
+        VARIANT: "3.8"
+        # Optional Node.js version to install
+        NODE_VERSION: "16"
+    environment:
+      DB_DRIVER: "postgresql"
+      DB_USER: "admin"
+      DB_PASSWORD: "postgres"
+
+    volumes:
+      - ..:/workspaces/rasa:cached
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity
+
+    # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
+    network_mode: service:db
+    # Uncomment the next line to use a non-root user for all processes.
+    # user: vscode
+
+  db:
+    image: "bitnami/postgresql:11.15.0"
+    restart: unless-stopped
+    volumes:
+      - postgres-data:/bitnami/postgresql
+    environment:
+      POSTGRESQL_USERNAME: admin
+      POSTGRESQL_DATABASE: rasa
+      POSTGRESQL_PASSWORD: postgres
+
+  duckling:
+    restart: unless-stopped
+    image: "rasa/duckling:0.2.0.2"
+    expose:
+      - "8000"
+    command: ["duckling-example-exe", "--no-access-log", "--no-error-log"]
+
+  redis:
+    restart: unless-stopped
+    image: "bitnami/redis:6.2.7"
+    environment:
+      REDIS_PASSWORD: "redis"
+    expose:
+      - "6379"
+
+volumes:
+  postgres-data: null

AI/.dockerignore

@@ -0,0 +1,11 @@
+docker*
+docs
+.git*
+**/*.pyc
+**/__pycache__
+!docker/configs
+rasa/tests
+rasa/scripts
+data/
+examples/
+docker-data/*

AI/.env

@@ -0,0 +1 @@
+TIMES=2

AI/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

AI/.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,2 @@
+<!-- IF YOU ARE ASKING A USAGE QUESTION (E.G. "HOW DO I DO XYZ") PLEASE POST
+YOUR QUESTION ON https://forum.rasa.com INSTEAD -->

AI/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Create a report to help us reproduce and fix the issue
+---
+
+**Before submitting a bug, please make sure the issue hasn't been already addressed by searching through the [FAQs](https://ai.meta.com/llama/faq/) and [existing/past issues](https://github.com/facebookresearch/llama/issues)**
+
+## Describe the bug
+<Please provide a clear and concise description of what the bug is. If relevant, please include a _minimal_ (least lines of code necessary) _reproducible_ (running this will give us the same result as you get) code snippet. Make sure to include the relevant imports.>
+
+### Minimal reproducible example
+<Remember to wrap the code in ```` ```triple-quotes blocks``` ````>
+
+```python
+# sample code to repro the bug
+```
+
+### Output
+<Remember to wrap the output in ```` ```triple-quotes blocks``` ````>
+
+```
+<paste stacktrace and other outputs here>
+```
+
+## Runtime Environment
+- Model: [eg: `meta-llama-3-8b-instruct`]
+- Using via huggingface?: [yes/no]
+- OS: [eg. Linux/Ubuntu, Windows]
+- GPU VRAM: 
+- Number of GPUs:
+- GPU Make: [eg: Nvidia, AMD, Intel]
+
+**Additional context**
+Add any other context about the problem or environment here.

AI/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Bug Report
+    url: https://rasa-open-source.atlassian.net/browse/OSS
+    about: Create a report to help us improve https://rasa-open-source.atlassian.net/browse/OSS
+  - name: Feature request
+    url: https://rasa-open-source.atlassian.net/browse/OSS
+    about: Suggest an idea on how to improve Rasa https://rasa-open-source.atlassian.net/browse/OSS
+  - name: Ask a question
+    url: https://forum.rasa.com/
+    about: If you have a "How do I?" question please ask in the forum https://forum.rasa.com

AI/.github/PULL_REQUEST_AUTOMATIC_TEMPLATE.md

@@ -0,0 +1,10 @@
+:bulb: This pull request was created automatically to merge a release branch back into the `main` branch.
+
+The changes you see here should have already been reviewed by someone, and shouldn't need an extra
+review. Nonetheless, if you notice something that needs to be addressed, please reach out to the person
+responsible for the original changes. In case additional changes need to be made, they need to target the release branch
+(not this pull request nor `main`).
+
+:auto_rickshaw: This PR should be merged automatically once it has been approved. If it doesn't happen:
+- [ ] Handle merge conflicts
+- [ ] Fix build errors

AI/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,8 @@
+**Proposed changes**:
+- ...
+
+**Status (please check what you already did)**:
+- [ ] added some tests for the functionality
+- [ ] updated the documentation
+- [ ] updated the changelog (please check [changelog](https://github.com/RasaHQ/rasa/tree/main/changelog) for instructions)
+- [ ] reformat files using `black` (please check [Readme](https://github.com/RasaHQ/rasa#code-style) for instructions)

AI/.github/change_filters.yml

@@ -0,0 +1,28 @@
+backend:
+  - 'pyproject.toml'
+  - 'poetry.lock'
+  - 'rasa/**/*'
+  - 'tests/**/*'
+  - 'data/**/*'
+  - 'examples/**/*'
+  - 'Makefile'
+  - '.github/workflows/continous-integration.yml'
+  - '.github/workflows/security-scans.yml'
+
+docker:
+  - 'pyproject.toml'
+  - 'poetry.lock'
+  - 'rasa/**/*'
+  - 'docker/**/*'
+  - 'Makefile'
+
+docs:
+  - 'docs/**/*'
+  - 'changelog/*'
+  - 'CHANGELOG.mdx'
+  - 'tests/docs/*'
+  - 'data/**/*'
+  - 'examples/**/*'
+  - 'Makefile'
+  - '.github/workflows/documentation.yml'
+  - '.github/workflows/ci-docs-tests.yml'

AI/.github/configs/mr-test-example.yaml

@@ -0,0 +1,49 @@
+## Example configuration
+#################### syntax #################
+## include:
+##   - dataset: ["<dataset_name>"]
+##     config: ["<configuration_name>"]
+#
+## Example:
+## include:
+##  - dataset: ["Carbon Bot"]
+##    config: ["Sparse + DIET(bow) + ResponseSelector(bow)"]
+#
+## Shortcut:
+## You can use the "all" shortcut to include all available configurations or datasets
+#
+## Example: Use the "Sparse + EmbeddingIntent + ResponseSelector(bow)" configuration
+## for all available datasets
+## include:
+##  - dataset: ["all"]
+##    config: ["Sparse + DIET(bow) + ResponseSelector(bow)"]
+#
+## Example: Use all available configurations for the "Carbon Bot" and "Sara" datasets
+## and for the "Hermit" dataset use the "Sparse + DIET + ResponseSelector(T2T)" and
+## "BERT + DIET + ResponseSelector(T2T)" configurations:
+## include:
+##  - dataset: ["Carbon Bot", "Sara"]
+##    config: ["all"]
+##  - dataset: ["Hermit"]
+##    config: ["Sparse + DIET(seq) + ResponseSelector(t2t)", "BERT + DIET(seq) + ResponseSelector(t2t)"]
+#
+## Example: Define a branch name to check-out for a dataset repository. Default branch is 'main'
+## dataset_branch: "test-branch"
+## include:
+##  - dataset: ["Carbon Bot", "Sara"]
+##    config: ["all"]
+#
+## Example: Define number of repetitions. This will inform how often to repeat all runs defined in the include section. Default is 1
+## num_repetitions: 2
+## include:
+##  - dataset: ["Carbon Bot", "Sara"]
+##    config: ["Sparse + DIET(seq) + ResponseSelector(t2t)"]
+##
+## Shortcuts:
+## You can use the "all" shortcut to include all available configurations or datasets.
+## You can use the "all-nlu" shortcut to include all available NLU configurations or datasets.
+## You can use the "all-core" shortcut to include all available core configurations or datasets.
+
+include:
+  - dataset: ["Carbon Bot"]
+    config: ["Sparse + DIET(bow) + ResponseSelector(bow)"]

AI/.github/configs/mr-test-schedule.json

@@ -0,0 +1,3 @@
+{
+  "body": "```yml\r\ninclude:\r\n  - dataset: [\"all\"]\r\n    config: [\"all\"]\r\n```"
+}

AI/.github/configs/tf-cuda.json

@@ -0,0 +1,25 @@
+{
+  "default_image_tag": "latest",
+  "config": [
+    {
+      "TF": "2.3",
+      "IMAGE_TAG": "cuda-10.1-cudnn7"
+    },
+    {
+      "TF": "2.5",
+      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
+    },
+    {
+      "TF": "2.6",
+      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
+    },
+    {
+      "TF": "2.7",
+      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
+    },
+    {
+      "TF": "2.11",
+      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
+    }
+  ]
+}

AI/.github/dependabot.yml

@@ -0,0 +1,34 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: weekly
+      time: "13:00"
+    pull-request-branch-name:
+      separator: "-"
+    open-pull-requests-limit: 10
+    labels:
+      - type:dependencies
+      - release:main
+    ignore:
+      - dependency-name: prompt-toolkit
+        versions:
+          - "> 2.0.10"
+      - dependency-name: pytest-asyncio
+        versions:
+          - "> 0.10.0"
+
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+      day: monday
+      time: "12:00"
+    pull-request-branch-name:
+      separator: "-"
+    open-pull-requests-limit: 10
+    reviewers:
+      - RasaHQ/infrastructure-squad
+    labels:
+      - type:dependencies

AI/.github/matchers/flake8-error-matcher.json

@@ -0,0 +1,17 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "flake8-error",
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "^([^:]+):(\\d+):(\\d+):\\s+([DCFNWE]\\d+\\s+.+)$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "message": 4
+        }
+      ]
+    }
+  ]
+}

AI/.github/no-response.yml

@@ -0,0 +1,12 @@
+# Configuration for probot-no-response - https://github.com/probot/no-response
+
+# Number of days of inactivity before an Issue is closed for lack of response
+daysUntilClose: 14
+# Label requiring a response
+responseRequiredLabel: status:more-details-needed
+# Comment to post when closing an Issue for lack of response. Set to `false` to disable
+closeComment: >
+  This issue has been automatically closed because there has been no response
+  to our request for more information from the original author. Without this,
+  we don't have enough information to help you. Please comment below with the
+  requested information if you still need help.

AI/.github/poetry_version.txt

@@ -0,0 +1,2 @@
+# The poetry version is stored in a separate file due to the https://github.com/python-poetry/poetry/issues/3316
+poetry-version=1.4.2

AI/.github/runner/github-runner-deployment.yaml.tmpl

@@ -0,0 +1,74 @@
+# GitHub Runner deployment - uses to deploy a github runner
+# which is used by the CI for model regression tests
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: github-runner-{{getenv "GITHUB_RUN_ID"}}
+  namespace: github-runner
+  labels:
+    app: github-runner
+    pod: github-runner-{{getenv "GITHUB_RUN_ID"}}
+spec:
+  replicas: {{getenv "NUM_REPLICAS" "1"}}
+  selector:
+    matchLabels:
+      app: github-runner
+      pod: github-runner-{{getenv "GITHUB_RUN_ID"}}
+  template:
+    metadata:
+      labels:
+        app: github-runner
+        pod: github-runner-{{getenv "GITHUB_RUN_ID"}}
+    spec:
+      priorityClassName: high-priority
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 720
+      containers:
+      - name: github-runner
+        image: {{getenv "GH_RUNNER_IMAGE"}}:{{getenv "GH_RUNNER_IMAGE_TAG" "latest"}}
+        imagePullPolicy: Always
+        livenessProbe:
+          initialDelaySeconds: 30
+          periodSeconds: 15
+          failureThreshold: 3
+          exec:
+            command:
+            - /bin/bash
+            - -c
+            - "if [[ `curl -sX GET -H \"Authorization: token ${GITHUB_PAT}\" \
+              https://api.github.com/repos/${GITHUB_OWNER}/${GITHUB_REPOSITORY}/actions/runners | \
+              jq -r '.runners[] | select(.name == \"'${POD_NAME}'\") | .status'` == \"offline\" ]]; then \
+              echo \"The GitHub API returns offline status for the ${POD_NAME} runner\" && exit 1; fi"
+        resources:
+          limits:
+            nvidia.com/gpu: 1
+          requests:
+            nvidia.com/gpu: 1
+            memory: 10G
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        # RUNNER_LABELS - defines labels
+        # with which a github-runner will be registered
+        - name: RUNNER_LABELS
+          value: "self-hosted,gpu,kubernetes,{{getenv "GITHUB_RUN_ID"}}"
+        # GITHUB_OWNER - a name of the repository owner
+        - name: GITHUB_OWNER
+          valueFrom:
+            secretKeyRef:
+              name: github-rasa
+              key: owner
+        # GITHUB_REPOSITORY - a name of the repository
+        - name: GITHUB_REPOSITORY
+          valueFrom:
+            secretKeyRef:
+              name: github-rasa
+              key: repository
+        # GITHUB_PAT - Personal Access Token
+        - name: GITHUB_PAT
+          valueFrom:
+            secretKeyRef:
+              name: github-rasa
+              key: pat

AI/.github/scripts/download_pretrained.py

@@ -0,0 +1,120 @@
+import argparse
+import logging
+import time
+from typing import List, NamedTuple, Optional, Text
+
+from transformers import AutoTokenizer, TFAutoModel
+
+import rasa.shared.utils.io
+from rasa.nlu.utils.hugging_face.registry import (
+    model_weights_defaults,
+    model_class_dict,
+)
+
+logger = logging.getLogger(__name__)
+
+COMP_NAME = "LanguageModelFeaturizer"
+DEFAULT_MODEL_NAME = "bert"
+
+
+class LmfSpec(NamedTuple):
+    """Holds information about the LanguageModelFeaturizer."""
+
+    model_name: Text
+    model_weights: Text
+    cache_dir: Optional[Text] = None
+
+
+def get_model_name_and_weights_from_config(
+    config_path: str,
+) -> List[LmfSpec]:
+    config = rasa.shared.utils.io.read_config_file(config_path)
+    logger.info(config)
+    steps = config.get("pipeline", [])
+
+    # Look for LanguageModelFeaturizer steps
+    steps = list(filter(lambda x: x["name"] == COMP_NAME, steps))
+
+    lmf_specs = []
+    for lmfeat_step in steps:
+        if "model_name" not in lmfeat_step:
+            if "model_weights" in lmfeat_step:
+                model_weights = lmfeat_step["model_weights"]
+                raise KeyError(
+                    "When model_name is not given, then model_weights cannot be set. "
+                    f"Here, model_weigths is set to {model_weights}"
+                )
+            model_name = DEFAULT_MODEL_NAME
+            model_weights = model_weights_defaults[DEFAULT_MODEL_NAME]
+        else:
+            model_name = lmfeat_step["model_name"]
+
+            if model_name not in model_class_dict:
+                raise KeyError(
+                    f"'{model_name}' not a valid model name. Choose from "
+                    f"{list(model_class_dict.keys())!s} or create"
+                    f"a new class inheriting from this class to support your model."
+                )
+
+            model_weights = lmfeat_step.get("model_weights")
+            if not model_weights:
+                logger.info(
+                    f"Model weights not specified. Will choose default model "
+                    f"weights: {model_weights_defaults[model_name]}"
+                )
+                model_weights = model_weights_defaults[model_name]
+        cache_dir = lmfeat_step.get("cache_dir", None)
+        lmf_specs.append(LmfSpec(model_name, model_weights, cache_dir))
+
+    return lmf_specs
+
+
+def instantiate_to_download(comp: LmfSpec) -> None:
+    """Instantiates Auto class instances, but only to download."""
+
+    _ = AutoTokenizer.from_pretrained(comp.model_weights, cache_dir=comp.cache_dir)
+    logger.info("Done with AutoTokenizer, now doing TFAutoModel")
+    _ = TFAutoModel.from_pretrained(comp.model_weights, cache_dir=comp.cache_dir)
+
+
+def download(config_path: str):
+    lmf_specs = get_model_name_and_weights_from_config(config_path)
+
+    if not lmf_specs:
+        logger.info(f"No {COMP_NAME} found, therefore, skipping download")
+        return
+
+    for lmf_spec in lmf_specs:
+        logger.info(
+            f"model_name: {lmf_spec.model_name}, "
+            f"model_weights: {lmf_spec.model_weights}, "
+            f"cache_dir: {lmf_spec.cache_dir}"
+        )
+        start = time.time()
+
+        instantiate_to_download(lmf_spec)
+
+        duration_in_sec = time.time() - start
+        logger.info(f"Instantiating Auto classes takes {duration_in_sec:.2f}seconds")
+
+
+def create_argument_parser() -> argparse.ArgumentParser:
+    """Downloads pretrained models, i.e., Huggingface weights."""
+    parser = argparse.ArgumentParser(
+        description="Downloads pretrained models, i.e., Huggingface weights, "
+        "e.g. path to bert_diet_responset2t.yml"
+    )
+    parser.add_argument(
+        "-c",
+        "--config",
+        type=str,
+        required=True,
+        help="The path to the config yaml file.",
+    )
+    return parser
+
+
+if __name__ == "__main__":
+    arg_parser = create_argument_parser()
+    cmdline_args = arg_parser.parse_args()
+    download(cmdline_args.config)

AI/.github/scripts/mr_generate_summary.py

@@ -0,0 +1,55 @@
+# Collect the results of the various model test runs which are done as part of
+# the model regression CI pipeline and dump them as a single file artifact.
+# This artifact will the then be published at the end of the tests.
+from collections import defaultdict
+import json
+import os
+from pathlib import Path
+from typing import Dict, List
+
+
+def combine_result(
+    result1: Dict[str, dict], result2: Dict[str, Dict[str, Dict]]
+) -> Dict[str, Dict[str, List]]:
+    """Combines 2 result dicts to accumulated dict of the same format.
+
+    Args:
+        result1: dict of key: dataset, value: (dict of key: config, value: list of res)
+                 Example: {
+                              "Carbon Bot": {
+                                  "Sparse + DIET(bow) + ResponseSelector(bow)": [{
+                                      "Entity Prediction": {
+                                          "macro avg": {
+                                              "f1-score": 0.88,
+                                          }
+                                      },
+                                      "test_run_time": "47s",
+                                  }]
+                              }
+                          }
+        result2: dict of key: dataset, value: (dict of key: config, value: list of res)
+
+    Returns:
+        dict of key: dataset, and value: (dict of key: config value: list of results)
+    """
+    combined_dict = defaultdict(lambda: defaultdict(list))
+    for new_dict in [result1, result2]:
+        for dataset, results_for_dataset in new_dict.items():
+            for config, res in results_for_dataset.items():
+                for res_dict in res:
+                    combined_dict[dataset][config].append(res_dict)
+    return combined_dict
+
+
+if __name__ == "__main__":
+    data = {}
+    reports_dir = Path(os.environ["REPORTS_DIR"])
+    reports_paths = list(reports_dir.glob("*/report.json"))
+
+    for report_path in reports_paths:
+        report_dict = json.load(open(report_path))
+        data = combine_result(data, report_dict)
+
+    summary_file = os.environ["SUMMARY_FILE"]
+    with open(summary_file, "w") as f:
+        json.dump(data, f, sort_keys=True, indent=2)

AI/.github/scripts/mr_publish_results.py

@@ -0,0 +1,293 @@
+# Send model regression test results to Datadog
+# with a summary of all test results.
+# Also write them into a report file.
+import copy
+import datetime
+import json
+import os
+from typing import Any, Dict, List, Text, Tuple
+
+from datadog_api_client.v1 import ApiClient, Configuration
+from datadog_api_client.v1.api.metrics_api import MetricsApi
+from datadog_api_client.v1.model.metrics_payload import MetricsPayload
+from datadog_api_client.v1.model.point import Point
+from datadog_api_client.v1.model.series import Series
+
+DD_ENV = "rasa-regression-tests"
+DD_SERVICE = "rasa"
+METRIC_RUNTIME_PREFIX = "rasa.perf.benchmark."
+METRIC_ML_PREFIX = "rasa.perf.ml."
+CONFIG_REPOSITORY = "training-data"
+
+TASK_MAPPING = {
+    "intent_report.json": "intent_classification",
+    "CRFEntityExtractor_report.json": "entity_prediction",
+    "DIETClassifier_report.json": "entity_prediction",
+    "response_selection_report.json": "response_selection",
+    "story_report.json": "story_prediction",
+}
+
+METRICS = {
+    "test_run_time": "TEST_RUN_TIME",
+    "train_run_time": "TRAIN_RUN_TIME",
+    "total_run_time": "TOTAL_RUN_TIME",
+}
+
+MAIN_TAGS = {
+    "config": "CONFIG",
+    "dataset": "DATASET_NAME",
+}
+
+OTHER_TAGS = {
+    "config_repository_branch": "DATASET_REPOSITORY_BRANCH",
+    "dataset_commit": "DATASET_COMMIT",
+    "accelerator_type": "ACCELERATOR_TYPE",
+    "type": "TYPE",
+    "index_repetition": "INDEX_REPETITION",
+    "host_name": "HOST_NAME",
+}
+
+GIT_RELATED_TAGS = {
+    "pr_id": "PR_ID",
+    "pr_url": "PR_URL",
+    "github_event": "GITHUB_EVENT_NAME",
+    "github_run_id": "GITHUB_RUN_ID",
+    "github_sha": "GITHUB_SHA",
+    "workflow": "GITHUB_WORKFLOW",
+}
+
+
+def create_dict_of_env(name_to_env: Dict[Text, Text]) -> Dict[Text, Text]:
+    return {name: os.environ[env_var] for name, env_var in name_to_env.items()}
+
+
+def _get_is_external_and_dataset_repository_branch() -> Tuple[bool, Text]:
+    is_external = os.environ["IS_EXTERNAL"]
+    dataset_repository_branch = os.environ["DATASET_REPOSITORY_BRANCH"]
+    if is_external.lower() in ("yes", "true", "t", "1"):
+        is_external_flag = True
+        dataset_repository_branch = os.environ["EXTERNAL_DATASET_REPOSITORY_BRANCH"]
+    else:
+        is_external_flag = False
+    return is_external_flag, dataset_repository_branch
+
+
+def prepare_datasetrepo_and_external_tags() -> Dict[Text, Any]:
+    is_external, dataset_repo_branch = _get_is_external_and_dataset_repository_branch()
+    return {
+        "dataset_repository_branch": dataset_repo_branch,
+        "external_dataset_repository": is_external,
+    }
+
+
+def prepare_dsrepo_and_external_tags_as_str() -> Dict[Text, Text]:
+    return {
+        "dataset_repository_branch": os.environ["DATASET_REPOSITORY_BRANCH"],
+        "external_dataset_repository": os.environ["IS_EXTERNAL"],
+    }
+
+
+def transform_to_seconds(duration: Text) -> float:
+    """Transform string (with hours, minutes, and seconds) to seconds.
+
+    Args:
+        duration: Examples: '1m27s', '1m27.3s', '27s', '1h27s', '1h1m27s'
+
+    Raises:
+        Exception: If the input is not supported.
+
+    Returns:
+        Duration converted in seconds.
+    """
+    h_split = duration.split("h")
+    if len(h_split) == 1:
+        rest = h_split[0]
+        hours = 0
+    else:
+        hours = int(h_split[0])
+        rest = h_split[1]
+    m_split = rest.split("m")
+    if len(m_split) == 2:
+        minutes = int(m_split[0])
+        seconds = float(m_split[1].rstrip("s"))
+    elif len(m_split) == 1:
+        minutes = 0
+        seconds = float(m_split[0].rstrip("s"))
+    else:
+        raise Exception(f"Unsupported duration: {duration}")
+    overall_seconds = hours * 60 * 60 + minutes * 60 + seconds
+    return overall_seconds
+
+
+def prepare_ml_metric(result: Dict[Text, Any]) -> Dict[Text, float]:
+    """Converts a nested result dict into a list of metrics.
+
+    Args:
+        result: Example
+            {'accuracy': 1.0,
+             'weighted avg': {
+                'precision': 1.0, 'recall': 1.0, 'f1-score': 1.0, 'support': 28
+             }
+            }
+
+    Returns:
+        Dict of metric name and metric value
+    """
+    metrics_ml = {}
+    result = copy.deepcopy(result)
+    result.pop("file_name", None)
+    task = result.pop("task", None)
+
+    for metric_name, metric_value in result.items():
+        if isinstance(metric_value, float):
+            metric_full_name = f"{task}.{metric_name}"
+            metrics_ml[metric_full_name] = float(metric_value)
+        elif isinstance(metric_value, dict):
+            for mname, mval in metric_value.items():
+                metric_full_name = f"{task}.{metric_name}.{mname}"
+                metrics_ml[metric_full_name] = float(mval)
+        else:
+            raise Exception(
+                f"metric_value {metric_value} has",
+                f"unexpected type {type(metric_value)}",
+            )
+    return metrics_ml
+
+
+def prepare_ml_metrics(results: List[Dict[Text, Any]]) -> Dict[Text, float]:
+    metrics_ml = {}
+    for result in results:
+        new_metrics_ml = prepare_ml_metric(result)
+        metrics_ml.update(new_metrics_ml)
+
+    return metrics_ml
+
+
+def prepare_datadog_tags() -> List[Text]:
+    tags = {
+        "env": DD_ENV,
+        "service": DD_SERVICE,
+        "branch": os.environ["BRANCH"],
+        "config_repository": CONFIG_REPOSITORY,
+        **prepare_dsrepo_and_external_tags_as_str(),
+        **create_dict_of_env(MAIN_TAGS),
+        **create_dict_of_env(OTHER_TAGS),
+        **create_dict_of_env(GIT_RELATED_TAGS),
+    }
+    tags_list = [f"{k}:{v}" for k, v in tags.items()]
+    return tags_list
+
+
+def send_to_datadog(results: List[Dict[Text, Any]]) -> None:
+    """Sends metrics to datadog."""
+    # Prepare
+    tags_list = prepare_datadog_tags()
+    timestamp = datetime.datetime.now().timestamp()
+    series = []
+
+    # Send metrics about runtime
+    metrics_runtime = create_dict_of_env(METRICS)
+    for metric_name, metric_value in metrics_runtime.items():
+        overall_seconds = transform_to_seconds(metric_value)
+        series.append(
+            Series(
+                metric=f"{METRIC_RUNTIME_PREFIX}{metric_name}.gauge",
+                type="gauge",
+                points=[Point([timestamp, overall_seconds])],
+                tags=tags_list,
+            )
+        )
+
+    # Send metrics about ML model performance
+    metrics_ml = prepare_ml_metrics(results)
+    for metric_name, metric_value in metrics_ml.items():
+        series.append(
+            Series(
+                metric=f"{METRIC_ML_PREFIX}{metric_name}.gauge",
+                type="gauge",
+                points=[Point([timestamp, float(metric_value)])],
+                tags=tags_list,
+            )
+        )
+
+    body = MetricsPayload(series=series)
+    with ApiClient(Configuration()) as api_client:
+        api_instance = MetricsApi(api_client)
+        response = api_instance.submit_metrics(body=body)
+        if response.get("status") != "ok":
+            print(response)
+
+
+def read_results(file: Text) -> Dict[Text, Any]:
+    with open(file) as json_file:
+        data = json.load(json_file)
+
+        keys = [
+            "accuracy",
+            "weighted avg",
+            "macro avg",
+            "micro avg",
+            "conversation_accuracy",
+        ]
+        result = {key: data[key] for key in keys if key in data}
+
+    return result
+
+
+def get_result(file_name: Text, file: Text) -> Dict[Text, Any]:
+    result = read_results(file)
+    result["file_name"] = file_name
+    result["task"] = TASK_MAPPING[file_name]
+    return result
+
+
+def send_all_to_datadog() -> None:
+    results = []
+    for dirpath, dirnames, files in os.walk(os.environ["RESULT_DIR"]):
+        for f in files:
+            if any(f.endswith(valid_name) for valid_name in TASK_MAPPING.keys()):
+                result = get_result(f, os.path.join(dirpath, f))
+                results.append(result)
+    send_to_datadog(results)
+
+
+def generate_json(file: Text, task: Text, data: dict) -> dict:
+    config = os.environ["CONFIG"]
+    dataset = os.environ["DATASET_NAME"]
+
+    if dataset not in data:
+        data = {dataset: {config: []}, **data}
+    elif config not in data[dataset]:
+        data[dataset] = {config: [], **data[dataset]}
+
+    assert len(data[dataset][config]) <= 1
+
+    data[dataset][config] = [
+        {
+            "config_repository": CONFIG_REPOSITORY,
+            **prepare_datasetrepo_and_external_tags(),
+            **create_dict_of_env(METRICS),
+            **create_dict_of_env(OTHER_TAGS),
+            **(data[dataset][config][0] if data[dataset][config] else {}),
+            task: read_results(file),
+        }
+    ]
+    return data
+
+
+def create_report_file() -> None:
+    data = {}
+    for dirpath, dirnames, files in os.walk(os.environ["RESULT_DIR"]):
+        for f in files:
+            if f not in TASK_MAPPING.keys():
+                continue
+
+            data = generate_json(os.path.join(dirpath, f), TASK_MAPPING[f], data)
+
+    with open(os.environ["SUMMARY_FILE"], "w") as f:
+        json.dump(data, f, sort_keys=True, indent=2)
+
+
+if __name__ == "__main__":
+    send_all_to_datadog()
+    create_report_file()

AI/.github/scripts/start_dd_agent.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+DD_API_KEY=$1
+ACCELERATOR_TYPE=$2
+NVML_INTERVAL_IN_SEC=${3:-15}  # 15 seconds are the default interval
+
+# Install Datadog system agent
+DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=$DD_API_KEY DD_SITE="datadoghq.eu" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
+DATADOG_YAML_PATH=/etc/datadog-agent/datadog.yaml
+sudo chmod 666 $DATADOG_YAML_PATH
+
+# Associate metrics with tags and env
+{
+    echo "env: rasa-regression-tests"
+    echo "tags:"
+    echo "- service:rasa"
+    echo "- accelerator_type:${ACCELERATOR_TYPE}"
+    echo "- dataset:${DATASET_NAME}"
+    echo "- config:${CONFIG}"
+    echo "- dataset_commit:${DATASET_COMMIT}"
+    echo "- branch:${BRANCH}"
+    echo "- github_sha:${GITHUB_SHA}"
+    echo "- pr_id:${PR_ID:-schedule}"
+    echo "- pr_url:${PR_URL:-schedule}"
+    echo "- type:${TYPE}"
+    echo "- dataset_repository_branch:${DATASET_REPOSITORY_BRANCH}"
+    echo "- external_dataset_repository:${IS_EXTERNAL:-none}"
+    echo "- config_repository:training-data"
+    echo "- config_repository_branch:${DATASET_REPOSITORY_BRANCH}"
+    echo "- workflow:${GITHUB_WORKFLOW:-none}"
+    echo "- github_run_id:${GITHUB_RUN_ID:-none}"
+    echo "- github_event:${GITHUB_EVENT_NAME:-none}"
+    echo "- index_repetition:${INDEX_REPETITION}"
+    echo "- host_name:${HOST_NAME}"
+    echo ""
+    echo "apm_config:"
+    echo "    enabled: true"
+    echo "process_config:"
+    echo "    enabled: false"
+    echo "use_dogstatsd: true"
+} >> $DATADOG_YAML_PATH
+
+# Enable system_core integration
+sudo mv /etc/datadog-agent/conf.d/system_core.d/conf.yaml.example /etc/datadog-agent/conf.d/system_core.d/conf.yaml
+
+if [[ "${ACCELERATOR_TYPE}" == "GPU" ]]; then
+    # Install and enable NVML integration
+    sudo datadog-agent integration --allow-root install -t datadog-nvml==1.0.1
+    sudo -u dd-agent -H /opt/datadog-agent/embedded/bin/pip3 install grpcio pynvml
+    NVML_CONF_FPATH="/etc/datadog-agent/conf.d/nvml.d/conf.yaml"
+    sudo mv "${NVML_CONF_FPATH}.example" ${NVML_CONF_FPATH}
+    if [[ "${NVML_INTERVAL_IN_SEC}" != 15 ]]; then
+        # Append a line to the NVML config file
+        sudo echo "    min_collection_interval: ${NVML_INTERVAL_IN_SEC}" | sudo tee -a ${NVML_CONF_FPATH} > /dev/null
+    fi
+fi
+
+# Apply changes
+sudo service datadog-agent stop
+
+# Restart agent (such that GPU/NVML metrics are collected)
+# Adusted code from /etc/init/datadog-agent.conf
+INSTALL_DIR="/opt/datadog-agent"
+AGENTPATH="$INSTALL_DIR/bin/agent/agent"
+PIDFILE="$INSTALL_DIR/run/agent.pid"
+AGENT_USER="dd-agent"
+LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64"
+sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $AGENTPATH run -p $PIDFILE"
+
+# Adusted code from /etc/init/datadog-agent-process.conf
+TRACE_AGENTPATH="$INSTALL_DIR/embedded/bin/trace-agent"
+TRACE_PIDFILE="$INSTALL_DIR/run/trace-agent.pid"
+sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $TRACE_PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $TRACE_AGENTPATH --config $DATADOG_YAML_PATH --pid $TRACE_PIDFILE"
+
+# Adusted code from /etc/init/datadog-agent-trace.conf
+PROCESS_AGENTPATH="$INSTALL_DIR/embedded/bin/process-agent"
+PROCESS_PIDFILE="$INSTALL_DIR/run/process-agent.pid"
+SYSTEM_PROBE_YAML="/etc/datadog-agent/system-probe.yaml"
+sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $PROCESS_PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $PROCESS_AGENTPATH --config=$DATADOG_YAML_PATH --sysprobe-config=$SYSTEM_PROBE_YAML --pid=$PROCESS_PIDFILE"

AI/.github/scripts/validate_cpu.py

@@ -0,0 +1,14 @@
+import sys
+
+import tensorflow as tf
+
+
+def check_gpu_not_available():
+    num_gpus = len(tf.config.list_physical_devices("GPU"))
+    print(f"Num GPUs Available: {num_gpus}")
+    if num_gpus > 0:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    check_gpu_not_available()

AI/.github/scripts/validate_gpus.py

@@ -0,0 +1,14 @@
+import sys
+
+import tensorflow as tf
+
+
+def check_gpu_available():
+    num_gpus = len(tf.config.list_physical_devices("GPU"))
+    print(f"Num GPUs Available: {num_gpus}")
+    if num_gpus <= 0:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    check_gpu_available()

AI/.github/stale.yml

@@ -0,0 +1,18 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 90
+# Label to use when marking an issue as stale
+staleLabel: stale
+
+pulls:
+  # Give more time before closing PRs
+  daysUntilClose: 21
+  # Comment to post when marking a PR as stale. Set to `false` to disable
+  markComment: >
+    This PR has been automatically marked as stale because it has not had
+    recent activity. It will be closed if no further activity occurs. Thank you
+    for your contributions.
+  # Comment to post when closing a stale PR. Set to `false` to disable
+  closeComment: >
+    This PR has been automatically closed due to inactivity. Please reopen
+    this PR or a new one if you plan to follow-up on it. Thank you for your
+    contributions.

AI/.github/templates/README.md

@@ -0,0 +1,63 @@
+# gomplate templates for GitHub Actions
+
+This document describes gomplate templates use for GitHub Actions.
+
+## Requirements
+
+You have to have installed [gomplate](https://docs.gomplate.ca/installing/) tool in order to render a template file.
+
+> gomplate is a template renderer which supports a growing list of datastores, such as: JSON (including EJSON - encrypted JSON), YAML, AWS EC2 metadata, BoltDB, Hashicorp Consul and Hashicorp Vault secrets.
+
+## Templates
+
+Below you can find a list of templates with their description and the commands to render them.
+
+### `configuration_variables.tmpl`
+
+The template maps dataset name and configuration name for the model regression tests into paths where files are located. As a result, the template returns two environment variables `DATASET` and `CONFIG` which contain paths to file/directory.
+
+#### How to run locally
+
+```shell
+gomplate -d mapping=<path_to_json_file_with_mapping> -f .github/templates/configuration_variables.tmpl
+```
+
+### `model_regression_test_config_comment.tmpl`
+
+The template returns a comment message which is used as a help description in a PR. The template reads the `.github/configs/mr-test-example.yaml` file and include it as example content.
+
+The help message is triggered by adding `status:model-regression-tests` label.
+Comment with a help message is added if a PR doesn't contain a comment with a configuration for the model regression tests.
+
+#### How to run locally
+
+```shell
+gomplate -f .github/templates/model_regression_test_config_comment.tmpl
+```
+
+The template uses the `GITHUB_ACTOR` environment variable, you have to export the variable before executing the command.
+
+### `model_regression_test_config_to_json.tmpl`
+
+The template reads an issue/a PR comment and transforms a YAML code block into JSON.
+
+#### How to run locally
+
+```shell
+gomplate -d github=https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ comment-id }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -f .github/templates/model_regression_test_config_to_json.tmpl
+```
+
+### `model_regression_test_results.tmpl`
+
+The template reads a file with a report (the report file is available as an artifact in the model regression tests workflow) and returns markdown table with a summary of tests.
+
+#### How to run locally
+
+```shell
+gomplate -d data=report.json -d results_main=report_main.json -f .github/templates/model_regression_test_results.tmpl
+```
+
+In order to be able to use the `.github/templates/model_regression_test_results.tmpl` template you need the following files:
+
+- `report.json` - the file with a report generated by the `CI - Model Regression` workflow run in a PR. The report is available to download as an artifact in the workflow related to the PR.
+- `report_main.json` - the file with a report generated by the `CI - Model Regression` workflow that is triggered on schedule event. A list of the workflows that you can download an artifact from, can be found [here](https://github.com/RasaHQ/rasa/actions?query=workflow%3A%22CI+-+Model+Regression%22+event%3Aschedule).

AI/.github/templates/configuration_variables.tmpl

@@ -0,0 +1,43 @@
+{{- /*
+
+The template maps dataset name and configuration name for the model
+regression tests into paths where files are located. As a result,
+the template returns two environment variables `DATASET` and `CONFIG`
+which contain paths to file/directory.
+
+ */ -}}
+{{- $mapping := (datasource "mapping") -}}
+{{- $dataset := (index $mapping.datasets (getenv "DATASET_NAME")) -}}
+{{- $config := $mapping.configurations -}}
+{{- if has $dataset "repository" }}
+export DATASET="{{ $dataset.repository }}"
+export IS_EXTERNAL="true"
+echo "::add-mask::{{ $dataset.repository }}"
+{{ if has $dataset "repository_branch" }}
+export EXTERNAL_DATASET_REPOSITORY_BRANCH="{{ $dataset.repository_branch }}"
+{{ else }}
+export EXTERNAL_DATASET_REPOSITORY_BRANCH="main"
+{{ end }}
+{{- else if has $dataset "path" }}
+export DATASET="{{ $dataset.path }}"
+export IS_EXTERNAL="false"
+echo "::add-mask::{{ $dataset.path }}"
+{{ end }}
+
+{{- if has $dataset "train" }}
+export TRAIN_DIR="{{ $dataset.train }}"
+{{ end }}
+{{- if has $dataset "test" }}
+export TEST_DIR="{{ $dataset.test }}"
+{{ end }}
+{{- if has $dataset "domain" }}
+export DOMAIN_FILE="{{ $dataset.domain }}"
+{{ end }}
+
+{{- if (has $config.nlu (getenv "CONFIG_NAME")) }}
+export CONFIG="{{ $dataset.language }}/nlu/{{ index $config.nlu (getenv "CONFIG_NAME") }}"
+echo "::add-mask::{{ $dataset.language }}/nlu/{{ index $config.nlu (getenv "CONFIG_NAME") }}"
+{{ else if (has $config.core (getenv "CONFIG_NAME")) }}
+export CONFIG="{{ $dataset.language }}/core/{{ index $config.core (getenv "CONFIG_NAME") }}"
+echo "::add-mask::{{ $dataset.language }}/core/{{ index $config.core (getenv "CONFIG_NAME") }}"
+{{ end -}}

AI/.github/templates/model_regression_test_config_comment.tmpl

@@ -0,0 +1,45 @@
+{{- /*
+
+The template returns a comment message which is used as a help description
+in a PR. The template reads the `.github/configs/mr-test-example.yaml` file
+and include it as example content.
+
+The help message is triggered by adding `status:model-regression-tests` label.
+Comment with a help message is added if a PR doesn't contain a comment
+with a configuration for the model regression tests.
+
+ */ -}}
+{{ define "check_available_configuration" -}}
+NLU
+{{- if has .dataset "domain" -}}
+, Core
+{{- end -}}
+{{- end -}}
+Hey @{{ .Env.GITHUB_ACTOR }}! :wave: To run model regression tests, comment with the `/modeltest` command and a configuration.
+
+_Tips :bulb:: The model regression test will be run on `push` events. You can re-run the tests by re-add `status:model-regression-tests` label or use a `Re-run jobs` button in Github Actions workflow._
+
+_Tips :bulb:: Every time when you want to change a configuration you should edit the comment with the previous configuration._
+
+You can copy this in your comment and customize:
+
+> /modeltest
+> ~~~yml
+>```yml
+>##########
+>## Available datasets
+>##########
+{{range (coll.Keys (datasource "mapping").datasets)}}># - "{{ . }}" ({{ template "check_available_configuration" (dict "dataset" (index (datasource "mapping").datasets .)) }}){{"\n"}}{{ end -}}
+>
+>##########
+>## Available NLU configurations
+>##########
+{{range (coll.Keys (datasource "mapping").configurations.nlu)}}># - "{{.}}"{{"\n"}}{{ end -}}
+>
+>##########
+>## Available Core configurations
+>##########
+{{range (coll.Keys (datasource "mapping").configurations.core)}}># - "{{.}}"{{"\n"}}{{ end -}}
+>
+{{range split (file.Read ".github/configs/mr-test-example.yaml") "\n"}}>{{.}}{{"\n"}}{{ end -}}
+>```

AI/.github/templates/model_regression_test_config_to_json.tmpl

@@ -0,0 +1,71 @@
+{{- /*
+
+The template reads an issue/a PR comment and transforms a YAML code block into JSON.
+
+*/ -}}
+{{ define "check_config_type" -}}
+{{- if has (datasource "mapping").configurations.nlu . -}}
+nlu
+{{- else if has (datasource "mapping").configurations.core . -}}
+core
+{{- end -}}
+{{- end -}}
+{{- $config := ((datasource "github").body | regexp.Find "```(?s)(.*)```" | regexp.ReplaceLiteral "```.*|\r" "" | yaml | toJSON | json) -}}
+{{- $num_repetitions := 1 -}}
+{{- if has $config "num_repetitions" -}}
+{{- $num_repetitions = $config.num_repetitions -}}
+{{- end -}}
+{"include":[
+{{- $inc := coll.Slice -}}
+{{- $dataset := coll.Slice -}}
+{{- range $pair := $config.include -}}
+{{- /* use all available datasets if value is equal to all */ -}}
+{{- if eq (index $pair.dataset 0) "all" -}}
+{{ $dataset = (coll.Keys (datasource "mapping").datasets) }}
+{{- else if eq (index $pair.dataset 0) "all-core" -}}
+{{- range $dataset_name, $dataset_spec := (datasource "mapping").datasets -}}
+{{- if has $dataset_spec "domain" -}}
+{{ $dataset = (coll.Append $dataset_name $dataset) -}}
+{{- end -}}
+{{- end -}}
+{{- else if eq (index $pair.dataset 0) "all-nlu" -}}
+{{- range $dataset_name, $dataset_spec := (datasource "mapping").datasets -}}
+{{- if not (has $dataset_spec "domain") -}}
+{{ $dataset = (coll.Append $dataset_name $dataset) -}}
+{{- end -}}
+{{- end -}}
+{{- else -}}
+{{- $dataset = $pair.dataset -}}
+{{- end -}}
+{{- range $index_dataset, $value_dataset := $dataset -}}
+{{- range $index_config, $value_config := $pair.config -}}
+{{ range $index_repetition, $element := (strings.Repeat $num_repetitions "x " | strings.Trim " " | strings.Split " ") }}
+{{- /* use all available configurations if value is equal to all */ -}}
+{{- if eq $value_config "all" -}}
+{{- range $config_type := (coll.Keys (datasource "mapping").configurations) -}}
+{{- range $config_name, $config_file := (index (datasource "mapping").configurations $config_type ) -}}
+
+{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $config_name "type" $config_type | toJSON) $inc) -}}
+{{- end -}}
+{{- end -}}
+{{- else if eq $value_config "all-core" -}}
+{{- range $config_name, $config_file := (datasource "mapping").configurations.core -}}
+{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $config_name "type" "core" | toJSON) $inc) -}}
+{{- end -}}
+{{- else if eq $value_config "all-nlu" -}}
+{{- range $config_name, $config_file := (datasource "mapping").configurations.nlu -}}
+{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $config_name "type" "nlu" | toJSON) $inc) -}}
+{{- end -}}
+{{- else -}}
+{{- if has (datasource "mapping").configurations.nlu $value_config -}}
+{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $value_config "type" "nlu" | toJSON) $inc) -}}
+{{- else if has (datasource "mapping").configurations.core $value_config -}}
+{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $value_config "type" "core" | toJSON) $inc) -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- join $inc "," -}}
+]}

AI/.github/templates/model_regression_test_read_dataset_branch.tmpl

@@ -0,0 +1,13 @@
+{{- /*
+
+The template reads a PR comment and gets the dataset branch for the training-data
+repository.
+
+ */ -}}
+{{- $config := ((datasource "github").body | regexp.Find "```(?s)(.*)```" | regexp.ReplaceLiteral "```.*|\r" "" | yaml | toJSON | json) -}}
+{{- $dataset_branch := "main" -}}
+{{- /* if a branch name for dataset repository is not defined use the main branch */ -}}
+{{- if has $config "dataset_branch" -}}
+{{- $dataset_branch = $config.dataset_branch -}}
+{{- end -}}
+export DATASET_BRANCH="{{ $dataset_branch }}"

AI/.github/templates/model_regression_test_results.tmpl

@@ -0,0 +1,159 @@
+{{- /*
+
+The template reads a file with a report (the report file is available
+as an artifact in the model regression tests workflow) and returns
+a markdown table with a summary of the tests.
+
+*/ -}}
+{{- /*
+
+The print_result_nlu template returns data depends on available fields.
+
+*/ -}}
+{{ define "print_result_nlu" -}}
+{{- if and (has (index .branch "micro avg") "f1-score") (has (index .main "micro avg") "f1-score") -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub (index (index .branch "micro avg") "f1-score")) }})
+{{- else if and (has .branch "accuracy") (has .main "accuracy") -}}
+{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" (.main.accuracy | math.Sub .branch.accuracy) }})
+{{- else if and (has .branch "accuracy") (has (index .main "micro avg") "f1-score") -}}
+{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub .branch.accuracy) }})
+{{- else if and (has (index .branch "micro avg") "f1-score") (has .main "accuracy") -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" (.main.accuracy | math.Sub (index (index .branch "micro avg") "f1-score")) }})
+{{- else if (has .branch "accuracy") -}}
+{{ printf "%.4f" .branch.accuracy }} (`no data`)
+{{- else if has (index .branch "micro avg") "f1-score" -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} (`no data`)
+{{- else -}}
+`no data`
+{{- end -}}
+{{- end -}}
+{{- /*
+
+The print_result_core template returns data depends on available fields.
+
+*/ -}}
+{{ define "print_result_core_micro_avg" -}}
+{{- if and (has (index .branch "micro avg") "f1-score") (has (index .main "micro avg") "f1-score") -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub (index (index .branch "micro avg") "f1-score")) }})
+{{- else if and (has .branch "accuracy") (has .main "accuracy") -}}
+{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" (.main.accuracy | math.Sub .branch.accuracy) }})
+{{- else if and (has .branch "accuracy") (has (index .main "micro avg") "f1-score") -}}
+{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub .branch.accuracy) }})
+{{- else if and (has (index .branch "micro avg") "f1-score") (has .main "accuracy") -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" (.main.accuracy | math.Sub (index (index .branch "micro avg") "f1-score")) }})
+{{- else if (has .branch "accuracy") -}}
+{{ printf "%.4f" .branch.accuracy }} (`no data`)
+{{- else if has (index .branch "micro avg") "f1-score" -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} (`no data`)
+{{- else -}}
+`no data`
+{{- end -}}
+{{- end -}}
+
+{{ define "print_result_core_conversation_accuracy" -}}
+{{- if and (has (index .branch "conversation_accuracy") "accuracy") (has (index .main "conversation_accuracy") "accuracy") -}}
+{{ printf "%.4f" (index (index .branch "conversation_accuracy") "accuracy") }} ({{ printf "%.2f" ((index (index .main "conversation_accuracy") "accuracy") | math.Sub (index (index .branch "conversation_accuracy") "accuracy")) }})
+{{- else if has (index .branch "conversation_accuracy") "accuracy" -}}
+{{ printf "%.4f" (index (index .branch "conversation_accuracy") "accuracy") }} (`no data`)
+{{- else -}}
+`no data`
+{{- end -}}
+{{- end -}}
+
+{{ define "print_table_nlu" }}
+{{- $available_types := (index .results_for_dataset | jsonpath `@..type`) -}}
+{{- if isKind "string" $available_types }}{{- $available_types = (index .results_for_dataset | jsonpath `@..type` | slice) -}}{{- end -}}
+{{- if has $available_types "nlu" -}}
+| Configuration | Intent Classification Micro F1 | Entity Recognition Micro F1 | Response Selection Micro F1 |
+|---------------|-----------------|-----------------|-------------------|
+{{ range $config_name, $config_data_array := .results_for_dataset -}}
+{{ range $config_data := $config_data_array }}
+{{- if eq $config_data.type "nlu" -}}
+| `{{ $config_name }}`<br> test: `{{ $config_data.test_run_time }}`, train: `{{ $config_data.train_run_time }}`, total: `{{ $config_data.total_run_time }}`|
+{{- if has $config_data "intent_classification" -}}
+{{- $intent_class_main := dict -}}
+{{- if has $.results_for_dataset_main $config_name -}}
+{{- $intent_class_main = (index (index $.results_for_dataset_main $config_name) 0).intent_classification -}}
+{{- end -}}
+{{- $intent_class := $config_data.intent_classification -}}
+{{ template "print_result_nlu" (dict "branch" $intent_class "main" $intent_class_main) }}|
+{{- else -}}
+`no data`|
+{{- end -}}
+{{- if has $config_data "entity_prediction" -}}
+{{- $entity_class_main := dict -}}
+{{- if has $.results_for_dataset_main $config_name -}}
+{{- $entity_class_main = (index (index $.results_for_dataset_main $config_name) 0).entity_prediction -}}
+{{- end -}}
+{{- $entity_class := $config_data.entity_prediction -}}
+{{ template "print_result_nlu" (dict "branch" $entity_class "main" $entity_class_main) }}|
+{{- else -}}
+`no data`|
+{{- end -}}
+{{- if has $config_data "response_selection" -}}
+{{- $response_class_main := dict -}}
+{{- if has $.results_for_dataset_main $config_name -}}
+{{- $response_class_main = (index (index $.results_for_dataset_main $config_name) 0).response_selection -}}
+{{- end -}}
+{{- $response_class := $config_data.response_selection -}}
+{{ template "print_result_nlu" (dict "branch" $response_class "main" $response_class_main) }}|
+{{- else -}}
+`no data`|
+{{- end }}
+{{end}}
+{{- end}}
+{{- end}}
+{{- end -}}
+{{- end -}}
+
+{{- define "print_table_core" -}}
+{{- $available_types := (index .results_for_dataset | jsonpath `@..type`) -}}
+{{- if isKind "string" $available_types }}{{- $available_types = (index .results_for_dataset | jsonpath `@..type` | slice) -}}{{- end -}}
+{{- if has $available_types "core" -}}
+| Dialog Policy Configuration | Action Level Micro Avg. F1 | Conversation Level Accuracy | Run Time Train | Run Time Test |
+|---------------|-----------------|-----------------|-------------------|-------------------|
+{{ range $config_name, $config_data_array := .results_for_dataset -}}
+{{ range $config_data := $config_data_array }}
+{{- if eq $config_data.type "core" -}}
+| `{{ $config_name }}` |
+{{- if has $config_data "story_prediction" -}}
+{{- $story_prediction_main := dict -}}
+{{- if has $.results_for_dataset_main $config_name -}}
+{{- $story_prediction_main = (index (index $.results_for_dataset_main $config_name) 0).story_prediction -}}
+{{- end -}}
+{{- $story_prediction := $config_data.story_prediction -}}
+{{ template "print_result_core_micro_avg" (dict "branch" $story_prediction "main" $story_prediction_main) }}|
+{{- else -}}
+`no data`|
+{{- end -}}
+{{- if has $config_data "story_prediction" -}}
+{{- $story_prediction_main := dict -}}
+{{- if has $.results_for_dataset_main $config_name -}}
+{{- $story_prediction_main = (index (index $.results_for_dataset_main $config_name) 0).story_prediction -}}
+{{- end -}}
+{{- $story_prediction := index $config_data.story_prediction -}}
+{{ template "print_result_core_conversation_accuracy" (dict "branch" $story_prediction "main" $story_prediction_main) }}|
+{{- else -}}
+`no data`|
+{{- end -}}
+`{{ $config_data.train_run_time }}`| `{{ $config_data.test_run_time }}`|
+{{ end }}
+{{- end}}
+{{- end}}
+{{- end -}}
+{{- end -}}
+
+{{- $results_main := (datasource "results_main") -}}
+{{ range $dataset, $results_for_dataset := (datasource "data")}}
+{{ $results_for_dataset_main := (index $results_main $dataset) -}}
+{{ $content_dicts := index $results_for_dataset (index (keys $results_for_dataset) 0) -}}
+{{ $one_content_dict := index $content_dicts 0 -}}
+{{- if ($one_content_dict).external_dataset_repository -}}
+Dataset: `{{$dataset}}`, Dataset repository branch: `{{ ($one_content_dict).dataset_repository_branch }}` (external repository), commit: `{{ ($one_content_dict).dataset_commit }}`
+Configuration repository branch: `{{ ($one_content_dict).config_repository_branch }}`
+{{ else -}}
+Dataset: `{{$dataset}}`, Dataset repository branch: `{{ ($one_content_dict).dataset_repository_branch }}`, commit: `{{ ($one_content_dict).dataset_commit }}`
+{{ end -}}
+{{ template "print_table_nlu" (dict "results_for_dataset" $results_for_dataset "results_for_dataset_main" $results_for_dataset_main) }}
+{{ template "print_table_core" (dict "results_for_dataset" $results_for_dataset "results_for_dataset_main" $results_for_dataset_main) }}
+{{- end }}

AI/.github/tests/test_data/bert_diet_response2t.yml

@@ -0,0 +1,23 @@
+# Configuration for Rasa NLU.
+# https://rasa.com/docs/rasa/nlu/components/
+language: en
+pipeline:
+  - name: WhitespaceTokenizer
+  - name: LanguageModelFeaturizer
+    alias: "lmf"
+  - name: RegexFeaturizer
+    alias: "rf"
+  - name: LexicalSyntacticFeaturizer
+    alias: "lsf"
+  - name: DIETClassifier
+    epochs: 50
+    random_seed: 42
+  - name: ResponseSelector
+    epochs: 100
+    num_transformer_layers: 2
+    transformer_size: 256
+    hidden_layers_size:
+      text: []
+      label: []
+    random_seed: 42
+    featurizers: ["lmf"]

AI/.github/tests/test_data/comment_body.json

@@ -0,0 +1,3 @@
+{
+  "body": "/modeltest\r\n\r\n```yml\r\ndataset_branch: \"test_dataset_branch\"\r\ninclude:\r\n - dataset: [\"financial-demo\"]\r\n   config: [\"TEST\"]\r\n ```\r\n\r\n<!-- comment-id:comment_configuration -->"
+}

AI/.github/tests/test_data/comment_body_no_dataset_branch.json

@@ -0,0 +1,3 @@
+{
+  "body": "/modeltest\r\n\r\n```yml\r\ninclude:\r\n - dataset: [\"financial-demo\"]\r\n   config: [\"TEST\"]\r\n ```\r\n\r\n<!-- comment-id:comment_configuration -->"
+}

AI/.github/tests/test_data/intent_report.json

@@ -0,0 +1,120 @@
+{
+	"search_transactions": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"greet": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"out_of_scope": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"thankyou": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"help": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"inform": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"goodbye": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"affirm": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 3,
+		"confused_with": {}
+	},
+	"pay_cc": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"check_balance": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 5,
+		"confused_with": {}
+	},
+	"deny": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"ask_transfer_charge": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"transfer_money": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 3,
+		"confused_with": {}
+	},
+	"check_recipients": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"check_earnings": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"accuracy": 1.0,
+	"macro avg": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 28
+	},
+	"weighted avg": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 28
+	}
+}

AI/.github/tests/test_data/report-on-schedule-2022-02-02.json

@@ -0,0 +1,303 @@
+{
+  "RasaHQ/financial-demo": {
+    "BERT + DIET(bow) + ResponseSelector(bow)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "35s",
+      "total_run_time": "2m2s",
+      "train_run_time": "1m28s",
+      "type": "nlu"
+    }],
+    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "55s",
+      "total_run_time": "2m8s",
+      "train_run_time": "1m14s",
+      "type": "nlu"
+    }],
+    "Rules + Memo + TED": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 48,
+          "total": 48,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 317
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 317
+        }
+      },
+      "test_run_time": "51s",
+      "total_run_time": "8m15s",
+      "train_run_time": "7m24s",
+      "type": "core"
+    }]
+  },
+  "RasaHQ/retail-demo": {
+    "BERT + DIET(bow) + ResponseSelector(bow)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.25,
+          "precision": 0.25,
+          "recall": 0.25,
+          "support": 6
+        },
+        "micro avg": {
+          "f1-score": 0.2857142857142857,
+          "precision": 1.0,
+          "recall": 0.16666666666666666,
+          "support": 6
+        },
+        "weighted avg": {
+          "f1-score": 0.16666666666666666,
+          "precision": 0.16666666666666666,
+          "recall": 0.16666666666666666,
+          "support": 6
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "macro avg": {
+          "f1-score": 0.8,
+          "precision": 0.8,
+          "recall": 0.85,
+          "support": 16
+        },
+        "micro avg": {
+          "f1-score": 0.8387096774193549,
+          "precision": 0.8666666666666667,
+          "recall": 0.8125,
+          "support": 16
+        },
+        "weighted avg": {
+          "f1-score": 0.8125,
+          "precision": 0.875,
+          "recall": 0.8125,
+          "support": 16
+        }
+      },
+      "test_run_time": "29s",
+      "total_run_time": "1m16s",
+      "train_run_time": "47s",
+      "type": "nlu"
+    }],
+    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.25,
+          "precision": 0.25,
+          "recall": 0.25,
+          "support": 6
+        },
+        "micro avg": {
+          "f1-score": 0.2857142857142857,
+          "precision": 1.0,
+          "recall": 0.16666666666666666,
+          "support": 6
+        },
+        "weighted avg": {
+          "f1-score": 0.16666666666666666,
+          "precision": 0.16666666666666666,
+          "recall": 0.16666666666666666,
+          "support": 6
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 0.875,
+        "macro avg": {
+          "f1-score": 0.8300000000000001,
+          "precision": 0.8166666666666667,
+          "recall": 0.85,
+          "support": 16
+        },
+        "weighted avg": {
+          "f1-score": 0.85,
+          "precision": 0.8333333333333333,
+          "recall": 0.875,
+          "support": 16
+        }
+      },
+      "test_run_time": "56s",
+      "total_run_time": "2m2s",
+      "train_run_time": "1m6s",
+      "type": "nlu"
+    }],
+    "Rules + Memo": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "conversation_accuracy": {
+          "accuracy": 0.8888888888888888,
+          "correct": 8,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 0.9663698541747322,
+          "precision": 1.0,
+          "recall": 0.946007696007696,
+          "support": 67
+        },
+        "micro avg": {
+          "f1-score": 0.9692307692307692,
+          "precision": 1.0,
+          "recall": 0.9402985074626866,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 0.9656317714563074,
+          "precision": 1.0,
+          "recall": 0.9402985074626866,
+          "support": 67
+        }
+      },
+      "test_run_time": "10s",
+      "total_run_time": "19s",
+      "train_run_time": "10s",
+      "type": "core"
+    }],
+    "Rules + Memo + TED": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 9,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        }
+      },
+      "test_run_time": "31s",
+      "total_run_time": "4m57s",
+      "train_run_time": "4m27s",
+      "type": "core"
+    }]
+  }
+}

AI/.github/tests/test_data/report_listformat_core.json

@@ -0,0 +1,70 @@
+{
+  "RasaHQ/retail-demo": {
+    "Rules + Memo + TED": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 9,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        }
+      },
+      "test_run_time": "31s",
+      "total_run_time": "4m57s",
+      "train_run_time": "4m27s",
+      "type": "core"
+    },
+    {
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 9,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        }
+      },
+      "test_run_time": "41s",
+      "total_run_time": "5m57s",
+      "train_run_time": "5m27s",
+      "type": "core"
+    }]
+  }
+}

AI/.github/tests/test_data/report_listformat_nlu.json

@@ -0,0 +1,98 @@
+{
+  "RasaHQ/financial-demo": {
+    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
+      "accelerator_type": "CPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "1m29s",
+      "total_run_time": "4m24s",
+      "train_run_time": "2m55s",
+      "type": "nlu"
+    },
+    {
+      "accelerator_type": "CPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "2m29s",
+      "total_run_time": "5m24s",
+      "train_run_time": "3m55s",
+      "type": "nlu"
+    }]
+  }
+}

AI/.github/tests/test_download_pretrained.py

@@ -0,0 +1,111 @@
+from copy import deepcopy
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+from ruamel.yaml import YAML
+
+sys.path.append(".github/scripts")
+import download_pretrained  # noqa: E402
+
+CONFIG_FPATH = Path(__file__).parent / "test_data" / "bert_diet_response2t.yml"
+
+
+def test_download_pretrained_lmf_exists_no_params():
+    lmf_specs = download_pretrained.get_model_name_and_weights_from_config(CONFIG_FPATH)
+    assert lmf_specs[0].model_name == "bert"
+    assert lmf_specs[0].model_weights == "rasa/LaBSE"
+
+
+def test_download_pretrained_lmf_exists_with_model_name():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(  # noqa: RUF015
+        filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps)
+    )[0]
+    step["model_name"] = "roberta"
+    step["cache_dir"] = "/this/dir"
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert lmf_specs[0].model_name == "roberta"
+    assert lmf_specs[0].model_weights == "roberta-base"
+    assert lmf_specs[0].cache_dir == "/this/dir"
+
+
+def test_download_pretrained_unknown_model_name():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(  # noqa: RUF015
+        filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps)
+    )[0]
+    step["model_name"] = "unknown"
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        with pytest.raises(KeyError):
+            download_pretrained.get_model_name_and_weights_from_config(fp.name)
+
+
+def test_download_pretrained_multiple_model_names():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(  # noqa: RUF015
+        filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps)
+    )[0]
+    step_new = deepcopy(step)
+    step_new["model_name"] = "roberta"
+    steps.append(step_new)
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert len(lmf_specs) == 2
+    assert lmf_specs[1].model_name == "roberta"
+
+
+def test_download_pretrained_with_model_name_and_nondefault_weight():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(  # noqa: RUF015
+        filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps)
+    )[0]
+    step["model_name"] = "bert"
+    step["model_weights"] = "bert-base-uncased"
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert lmf_specs[0].model_name == "bert"
+    assert lmf_specs[0].model_weights == "bert-base-uncased"
+
+
+def test_download_pretrained_lmf_doesnt_exists():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(  # noqa: RUF015
+        filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps)
+    )[0]
+    steps.remove(step)
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert len(lmf_specs) == 0

AI/.github/tests/test_model_regression_test_read_dataset_branch_tmpl.py

@@ -0,0 +1,27 @@
+import pathlib
+import subprocess
+import pytest
+from typing import Text
+
+TEMPLATE_FPATH = ".github/templates/model_regression_test_read_dataset_branch.tmpl"
+REPO_DIR = pathlib.Path("").absolute()
+TEST_DATA_DIR = str(pathlib.Path(__file__).parent / "test_data")
+DEFAULT_DATASET_BRANCH = "main"
+
+
+@pytest.mark.parametrize(
+    "comment_body_file,expected_dataset_branch",
+    [
+        ("comment_body.json", "test_dataset_branch"),
+        ("comment_body_no_dataset_branch.json", DEFAULT_DATASET_BRANCH),
+    ],
+)
+def test_read_dataset_branch(comment_body_file: Text, expected_dataset_branch: Text):
+    cmd = (
+        "gomplate "
+        f"-d github={TEST_DATA_DIR}/{comment_body_file} "
+        f"-f {TEMPLATE_FPATH}"
+    )
+    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
+    output = output.decode("utf-8").strip()
+    assert output == f'export DATASET_BRANCH="{expected_dataset_branch}"'

AI/.github/tests/test_model_regression_test_results_tmpl.py

@@ -0,0 +1,50 @@
+import pathlib
+import subprocess
+
+TEMPLATE_FPATH = ".github/templates/model_regression_test_results.tmpl"
+REPO_DIR = pathlib.Path("").absolute()
+TEST_DATA_DIR = str(pathlib.Path(__file__).parent / "test_data")
+
+
+def test_comment_nlu():
+    cmd = (
+        "gomplate "
+        f"-d data={TEST_DATA_DIR}/report_listformat_nlu.json "
+        f"-d results_main={TEST_DATA_DIR}/report-on-schedule-2022-02-02.json "
+        f"-f {TEMPLATE_FPATH}"
+    )
+    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
+    output = output.decode("utf-8")
+    expected_output = """
+Dataset: `RasaHQ/financial-demo`, Dataset repository branch: `fix-model-regression-tests` (external repository), commit: `52a3ad3eb5292d56542687e23b06703431f15ead`
+Configuration repository branch: `main`
+| Configuration | Intent Classification Micro F1 | Entity Recognition Micro F1 | Response Selection Micro F1 |
+|---------------|-----------------|-----------------|-------------------|
+| `BERT + DIET(seq) + ResponseSelector(t2t)`<br> test: `1m29s`, train: `2m55s`, total: `4m24s`|1.0000 (0.00)|0.8333 (0.00)|`no data`|
+| `BERT + DIET(seq) + ResponseSelector(t2t)`<br> test: `2m29s`, train: `3m55s`, total: `5m24s`|1.0000 (0.00)|0.8333 (0.00)|`no data`|
+
+
+"""  # noqa E501
+    assert output == expected_output
+
+
+def test_comment_core():
+    cmd = (
+        "gomplate "
+        f"-d data={TEST_DATA_DIR}/report_listformat_core.json "
+        f"-d results_main={TEST_DATA_DIR}/report-on-schedule-2022-02-02.json "
+        f"-f {TEMPLATE_FPATH}"
+    )
+    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
+    output = output.decode("utf-8")
+    expected_output = """
+Dataset: `RasaHQ/retail-demo`, Dataset repository branch: `fix-model-regression-tests` (external repository), commit: `8226b51b4312aa4d3723098cf6d4028feea040b4`
+Configuration repository branch: `main`
+
+| Dialog Policy Configuration | Action Level Micro Avg. F1 | Conversation Level Accuracy | Run Time Train | Run Time Test |
+|---------------|-----------------|-----------------|-------------------|-------------------|
+| `Rules + Memo + TED` |1.0000 (0.00)|1.0000 (0.00)|`4m27s`| `31s`|
+| `Rules + Memo + TED` |1.0000 (0.00)|1.0000 (0.00)|`5m27s`| `41s`|
+
+"""  # noqa E501
+    assert output == expected_output

AI/.github/tests/test_mr_generate_summary.py

@@ -0,0 +1,208 @@
+import sys
+
+sys.path.append(".github/scripts")
+from mr_generate_summary import combine_result  # noqa: E402
+
+
+RESULT1 = {
+    "financial-demo": {
+        "BERT + DIET(bow) + ResponseSelector(bow)": [
+            {
+                "Entity Prediction": {
+                    "macro avg": {
+                        "f1-score": 0.7333333333333333,
+                    }
+                },
+                "test_run_time": "47s",
+            }
+        ]
+    }
+}
+
+
+def test_same_ds_different_config():
+    result2 = {
+        "financial-demo": {
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        }
+    }
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ],
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ],
+        }
+    }
+
+    actual_combined = combine_result(RESULT1, result2)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(result2, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_different_ds_same_config():
+    result2 = {
+        "Carbon Bot": {
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        }
+    }
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ],
+        },
+        "Carbon Bot": {
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        },
+    }
+
+    actual_combined = combine_result(RESULT1, result2)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(result2, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_start_empty():
+    result2 = {}
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        }
+    }
+
+    actual_combined = combine_result(RESULT1, result2)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(result2, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_combine_result_repetition():
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+            ]
+        }
+    }
+
+    actual_combined = combine_result(RESULT1, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_combine_result_repetition_3times():
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+            ]
+        }
+    }
+
+    tmp_combined = combine_result(RESULT1, RESULT1)
+    actual_combined = combine_result(tmp_combined, RESULT1)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(RESULT1, tmp_combined)
+    assert actual_combined == expected_combined

AI/.github/tests/test_mr_publish_results.py

@@ -0,0 +1,132 @@
+import os
+from pathlib import Path
+import sys
+from unittest import mock
+
+sys.path.append(".github/scripts")
+from mr_publish_results import (  # noqa: E402
+    prepare_ml_metric,
+    prepare_ml_metrics,
+    transform_to_seconds,
+    generate_json,
+    prepare_datadog_tags,
+)
+
+EXAMPLE_CONFIG = "Sparse + BERT + DIET(seq) + ResponseSelector(t2t)"
+EXAMPLE_DATASET_NAME = "financial-demo"
+
+ENV_VARS = {
+    "BRANCH": "my-branch",
+    "PR_ID": "10927",
+    "PR_URL": "https://github.com/RasaHQ/rasa/pull/10856/",
+    "GITHUB_EVENT_NAME": "pull_request",
+    "GITHUB_RUN_ID": "1882718340",
+    "GITHUB_SHA": "abc",
+    "GITHUB_WORKFLOW": "CI - Model Regression",
+    "IS_EXTERNAL": "false",
+    "DATASET_REPOSITORY_BRANCH": "main",
+    "CONFIG": EXAMPLE_CONFIG,
+    "DATASET_NAME": EXAMPLE_DATASET_NAME,
+    "CONFIG_REPOSITORY_BRANCH": "main",
+    "DATASET_COMMIT": "52a3ad3eb5292d56542687e23b06703431f15ead",
+    "ACCELERATOR_TYPE": "CPU",
+    "TEST_RUN_TIME": "1m54s",
+    "TRAIN_RUN_TIME": "4m4s",
+    "TOTAL_RUN_TIME": "5m58s",
+    "TYPE": "nlu",
+    "INDEX_REPETITION": "0",
+    "HOST_NAME": "github-runner-2223039222-22df222fcd-2cn7d",
+}
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_generate_json():
+    f = Path(__file__).parent / "test_data" / "intent_report.json"
+    result = generate_json(f, task="intent_classification", data={})
+    assert isinstance(result[EXAMPLE_DATASET_NAME][EXAMPLE_CONFIG], list)
+
+    actual = result[EXAMPLE_DATASET_NAME][EXAMPLE_CONFIG][0]["intent_classification"]
+    expected = {
+        "accuracy": 1.0,
+        "weighted avg": {
+            "precision": 1.0,
+            "recall": 1.0,
+            "f1-score": 1.0,
+            "support": 28,
+        },
+        "macro avg": {"precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 28},
+    }
+    assert expected == actual
+
+
+def test_transform_to_seconds():
+    assert 87.0 == transform_to_seconds("1m27s")
+    assert 87.3 == transform_to_seconds("1m27.3s")
+    assert 27.0 == transform_to_seconds("27s")
+    assert 3627.0 == transform_to_seconds("1h27s")
+    assert 3687.0 == transform_to_seconds("1h1m27s")
+
+
+def test_prepare_ml_model_perf_metrics():
+    results = [
+        {
+            "macro avg": {
+                "precision": 0.8,
+                "recall": 0.8,
+                "f1-score": 0.8,
+                "support": 14,
+            },
+            "micro avg": {
+                "precision": 1.0,
+                "recall": 0.7857142857142857,
+                "f1-score": 0.88,
+                "support": 14,
+            },
+            "file_name": "DIETClassifier_report.json",
+            "task": "Entity Prediction",
+        },
+        {
+            "accuracy": 1.0,
+            "weighted avg": {
+                "precision": 1.0,
+                "recall": 1.0,
+                "f1-score": 1.0,
+                "support": 28,
+            },
+            "macro avg": {
+                "precision": 1.0,
+                "recall": 1.0,
+                "f1-score": 1.0,
+                "support": 28,
+            },
+            "file_name": "intent_report.json",
+            "task": "Intent Classification",
+        },
+    ]
+    metrics_ml = prepare_ml_metrics(results)
+    assert len(metrics_ml) == 17
+
+
+def test_prepare_ml_model_perf_metrics_simple():
+    result = {
+        "accuracy": 1.0,
+        "weighted avg": {"precision": 1, "recall": 1.0, "f1-score": 1, "support": 28},
+        "task": "Intent Classification",
+    }
+    metrics_ml = prepare_ml_metric(result)
+    assert len(metrics_ml) == 5
+
+    for _, v in metrics_ml.items():
+        assert isinstance(v, float)
+
+    key, value = "Intent Classification.accuracy", 1.0
+    assert key in metrics_ml and value == metrics_ml[key]
+
+    key, value = "Intent Classification.weighted avg.f1-score", 1.0
+    assert key in metrics_ml and value == metrics_ml[key]
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_prepare_datadog_tags():
+    tags_list = prepare_datadog_tags()
+    assert "dataset:financial-demo" in tags_list

AI/.github/tests/test_validate_gpus.py

@@ -0,0 +1,27 @@
+import os
+import sys
+from unittest import mock
+
+import pytest
+
+sys.path.append(".github/scripts")
+import validate_cpu  # noqa: E402
+import validate_gpus  # noqa: E402
+
+ENV_VARS = {
+    "CUDA_VISIBLE_DEVICES": "-1",
+}
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_validate_cpu_succeeds_when_there_are_no_gpus():
+    validate_cpu.check_gpu_not_available()
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_validate_gpus_exits_when_there_are_no_gpus():
+    # This unit test assumes that unit tests are run on a CPU
+    with pytest.raises(SystemExit) as pytest_wrapped_e:
+        validate_gpus.check_gpu_available()
+    assert pytest_wrapped_e.type == SystemExit
+    assert pytest_wrapped_e.value.code == 1

AI/.github/workflows/automatic-pr-update.yml

@@ -0,0 +1,22 @@
+name: Automatic PR Merger
+
+on:
+  push: {} # update PR when base branch is updated
+
+jobs:
+  # thats's all. single step is needed - if PR is mergeable according to
+  # branch protection rules it will be merged automatically
+  mergepal:
+    runs-on: ubuntu-22.04
+    if: github.repository == 'RasaHQ/rasa'
+
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+      - uses: rasahq/update-pr-branch@f7012036a6d5659cfbc37f180716963511e81f95
+        with:
+          token: ${{ secrets.UPDATE_BRANCH_PAT }}
+          # required parameter by original action -
+          # check is already done through protected branches so not needed for us
+          required_approval_count: 0
+          # update branch despite failing check runs
+          require_passed_checks: false