add documentation and model weights (#1)

- initial updates to repo and licensing (3b89e4303872073829a482c66fb4961c0af4da5c)
- preprocessing code, more in github (0f1419cf71677187f3962000806f6cbcf2c507b5)
- readme with image (1f07f22e4df48da1ef9656f5c3186a915ac3a3c3)
- model weights (6a2b66469aa57e99de4a73f3e752dcb3c4453683)
- error in readme (f66778411dca6f51f409a6038d8887695a526e88)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,15 @@
+.log/
+.vscode/
+__pycache__/
+assets_local/
+*/__pycache__/
+.venv/
+_t*.py 
+_*.yaml
+_develop_*.py
+outputs/
+jupyter/*
+*.csv 
+third_party/*
+dev_*.py
+.venv

LICENSE

@@ -0,0 +1,402 @@
+Attribution-NonCommercial-NoDerivatives 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+    wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More considerations
+     for the public:
+    wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution-NonCommercial-NoDerivatives 4.0
+International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial-NoDerivatives 4.0 International Public
+License ("Public License"). To the extent this Public License may be
+interpreted as a contract, You are granted the Licensed Rights in
+consideration of Your acceptance of these terms and conditions, and the
+Licensor grants You such rights in consideration of benefits the
+Licensor receives from making the Licensed Material available under
+these terms and conditions.
+
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+
+  c. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  d. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  e. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  f. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  g. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  h. NonCommercial means not primarily intended for or directed towards
+     commercial advantage or monetary compensation. For purposes of
+     this Public License, the exchange of the Licensed Material for
+     other material subject to Copyright and Similar Rights by digital
+     file-sharing or similar means is NonCommercial provided there is
+     no payment of monetary compensation in connection with the
+     exchange.
+
+  i. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  j. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  k. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part, for NonCommercial purposes only; and
+
+            b. produce and reproduce, but not Share, Adapted Material
+               for NonCommercial purposes only.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties, including when
+          the Licensed Material is used other than for NonCommercial
+          purposes.
+
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material, You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+          For the avoidance of doubt, You do not have permission under
+          this Public License to Share Adapted Material.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database for NonCommercial purposes
+     only and provided You do not Share Adapted Material;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material; and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.

README.md

@@ -1,3 +1,54 @@
 ---
-license: cc-by-nc-4.0
+language:
+- en
+license: cc-by-nc-nd-4.0
+tags:
+- cxr
+- ecg
+- echocardiogram
+- probabilistic modelling
+- multimodal
+- medical
+pipeline_tag: other
 ---
+
+# ProbMED: A Probabilistic Framework for Medical Multimodal Binding (ICCV 2025)
+Probabilistic Modality-Enhanced Diagnosis (ProbMED), a multi-modal Med-VLPM that employs probabilistic contrastive learning to model distributions over embeddings rather than fixed-point, deterministic estimates. ProbMED aligns four distinct modalities—chest X-rays, electrocardiograms, echocardiograms, and clinical text—into a unified probabilistic embedding space.
+
+<p align="center">
+  <img src="assets/github_highlevel.png" width="40%">
+</p>
+
+
+## Installation
+Clone the GitHub repository and install dependencies, instructions are found in the repo:
+```bash
+git clone [email protected]:mcintoshML/probMED.git
+cd probMED
+pip install -r requirements.txt
+```
+
+## Full Code Release
+The model weights and inference is available with this code base. 
+
+**We plan to release the full training and evaluation codebase upon the clinical journal submission to facilitate reproducibility, please stay tuned!**
+
+## License
+This work is licensed under the **Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License (CC BY-NC-ND 4.0)**.
+
+You may share this work for non-commercial purposes, with proper attribution, but you may not modify it or use it commercially.
+
+[![Creative Commons License](https://i.creativecommons.org/l/by-nc-nd/4.0/88x31.png)](https://creativecommons.org/licenses/by-nc-nd/4.0/)
+
+[View Full License Details](https://creativecommons.org/licenses/by-nc-nd/4.0/)
+
+## Citation
+If you use ProbMED in your research (ICCV 2025), please cite:
+```
+@article{gao2025probmed,
+  title={ProbMed: A Probabilistic Framework for Medical Multimodal Binding},
+  author={Gao, Yuan and Kim, Sangwook and You, Jianzhong and McIntosh, Chris},
+  journal={arXiv preprint arXiv:2509.25711},
+  year={2025}
+}
+```

mixinhelpers.py

@@ -0,0 +1,221 @@
+# For CXR
+import random
+
+import cv2
+import numpy as np
+import torch
+from PIL import Image
+from torchvision import transforms
+from transformers import BatchEncoding, PreTrainedTokenizer
+
+"""
+Mixin for all modalities, each mixin has:
+- preprocess function that takes in path or data and returns tensor
+- construct_input function that takes in tensor and returns dict with batch
+dimension for model input
+- key string for model input dict
+"""
+
+
+class ECHO_Mixin:
+    LOWER_YELLOW: list[int] = [20, 50, 50]
+    UPPER_YELLOW: list[int] = [100, 255, 255]
+    IMAGE_SIZE: tuple[int, int] = (224, 224)
+    NORM_MEAN: tuple[float, float, float] = (0.48145466, 0.4578275, 0.40821073)
+    NORM_STD: tuple[float, float, float] = (0.26862954, 0.26130258, 0.27577711)
+
+    ECHO_TRANSFORMS = transforms.Compose(
+        [
+            transforms.ToTensor(),  # Scaling into [0, 1]
+            transforms.Resize(IMAGE_SIZE),
+            transforms.Normalize(
+                mean=NORM_MEAN,
+                std=NORM_STD,
+            ),
+        ]
+    )
+    ECHO_KEY: str = "echo"
+
+    def grabimage(self, split: str, data: dict[str, np.ndarray]) -> np.ndarray:
+        """"""
+        if split == "train":
+            caseofinterest = random.choice(list(data.keys()))
+            imageindice = random.choice(list(range(data[caseofinterest].shape[0])))
+
+        else:
+            caseofinterest = random.choice(list(data.keys()))  # listofcases[0]
+            imageindice = 0
+        video = data[caseofinterest]
+        return self.extract_echoframe(imageindice, video)
+
+    def extract_echoframe(self, imageindice: int, video: np.ndarray) -> np.ndarray:
+        image = video[imageindice]
+        hsv_image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
+        lower_yellow = np.array(self.LOWER_YELLOW)  # Lower bound of yellow hue
+        upper_yellow = np.array(self.UPPER_YELLOW)  # Upper bound of yellow hue
+        mask = cv2.inRange(hsv_image, lower_yellow, upper_yellow)
+        image[mask > 0] = [0, 0, 0]
+        image = np.array(image, dtype=np.float32)
+        image -= image.min()
+        image /= image.max()
+        image *= 255
+
+        image = image
+        image = image[:, :, :]
+        image = image.astype(np.uint8)
+        return image
+
+    def preprocess_echoseries(
+        self, video_dict: dict[str, np.ndarray], split: str = "valid"
+    ) -> torch.Tensor:
+        """assumes inference mode"""
+        image = self.grabimage(split, video_dict)
+        if not isinstance(image, np.ndarray):
+            raise TypeError("Expected image to be a numpy ndarray")
+        pil_image = Image.fromarray(image)
+        transformed = self.ECHO_TRANSFORMS(pil_image)
+        if not isinstance(transformed, torch.Tensor):
+            transformed = transforms.ToTensor()(pil_image)
+        return transformed
+
+    def preprocess_single_echo(self, avi_path: str) -> torch.Tensor:
+        """assumes inference mode, opens AVI file and processes first frame
+        Output: image: torch.Tensor of shape (C, H, W)
+        """
+        cap = cv2.VideoCapture(avi_path)
+        success, frame = cap.read()
+        cap.release()
+        if not success or frame is None:
+            raise ValueError(f"Could not read frame from AVI file: {avi_path}")
+        image = self.extract_echoframe(0, np.array([frame]))  # process first frame
+        image = self.ECHO_TRANSFORMS(Image.fromarray(image))
+        if not isinstance(image, torch.Tensor):
+            image = torch.from_numpy(image)
+        return image
+
+
+# CXR
+class CXR_Mixin:
+    RESIZE: tuple[int, int] = (256, 256)
+    IMAGE_SIZE: tuple[int, int] = (224, 224)
+    NORM_MEAN: list[float] = [0.5862785803043838]
+    NORM_STD: list[float] = [0.27950088968644304]
+    VISION_KEY: str = "vision"
+    CXR_TRANSFORMS = transforms.Compose(
+        [
+            transforms.ToTensor(),  # Scaling into [0, 1]
+            transforms.Resize(RESIZE),
+            transforms.CenterCrop(IMAGE_SIZE),
+            transforms.Normalize(
+                mean=NORM_MEAN,
+                std=NORM_STD,
+            ),
+        ]
+    )
+
+    @staticmethod
+    def remove_border(pixel_array: np.ndarray) -> np.ndarray:
+        # Find where the image is not just background (0s)
+        coords = np.column_stack(np.where(pixel_array > 0))
+        x_min, y_min = coords.min(axis=0)
+        x_max, y_max = coords.max(axis=0)
+        # Crop the image
+        cropped_image = pixel_array[x_min:x_max, y_min:y_max]
+        return cropped_image
+
+    def preprocess_loaded_cxr(self, img: np.array) -> torch.Tensor:
+        cxr = self.remove_border(img)
+        # Convert grayscale image to 3-channel RGB
+        cxr = np.repeat(cxr[..., np.newaxis], 3, axis=-1)
+
+        cxr = Image.fromarray(cxr)
+        transformed = self.CXR_TRANSFORMS(cxr)
+        if not isinstance(transformed, torch.Tensor):
+            transformed = transforms.ToTensor()(cxr)
+        return transformed
+
+    def preprocess_single_cxr(self, image_path: str) -> torch.Tensor:
+        """assumes inference mode"""
+        with open(image_path, "rb") as fopen:
+            image = Image.open(fopen).convert("RGB")
+            image = np.array(image)[:, :, 0]  # convert to grayscale
+
+        cxr = self.preprocess_loaded_cxr(image)
+        return cxr
+
+
+class ECG_Mixin:
+    LENGTH: int = 1000
+    FREQUENCY: int = 100  # we assume 100Hz sampling rate
+    CHANNELS: int = 12
+    NORM_MEAN: float = 0.02547506
+    NORM_SCALE: float = 0.16486814
+    NORM_VAR: float = 0.0271815
+    ECG_KEY: str = "ecg"
+
+    def manual_standardize(self, x: np.ndarray) -> torch.Tensor:
+        """
+        Apply manual standardization to ECG or other data.
+        Equivalent to sklearn's StandardScaler with given constants.
+
+        Args:
+            x (np.ndarray): Input array of shape (12, 1000)
+        Returns:
+            torch.Tensor: Scaled array of the same shape
+        """
+        return torch.from_numpy((x - self.NORM_MEAN) / self.NORM_SCALE).float()
+
+    def check_ecg(self, ecg: np.ndarray) -> np.ndarray:
+        # Find where the image is not just background (0s)
+        if np.isnan(ecg).any() or np.isinf(ecg).any():
+            raise ValueError("ECG contains NaN or Inf values")
+        return ecg[:, : self.LENGTH]  # Truncate to first 1000 length (10 seconds at 100Hz)
+
+    def preprocess_single_ecg(self, ecg_path: str) -> torch.Tensor:
+        """assumes inference mode"""
+        # ecg is a np array path, assumes 12 channels
+        ecg = np.load(ecg_path)
+        if ecg.ndim == 2 and ecg.shape[0] != self.CHANNELS:
+            raise ValueError(f"Expected ECG with {self.CHANNELS} channels, got {ecg.shape[0]}")
+
+        ecg = self.check_ecg(ecg)
+        transformed = self.manual_standardize(ecg)
+
+        return transformed
+
+
+class Text_Mixin:
+    MODALITY_LIST: dict[str, str] = {"echo": "echocardiogram", "ecg": "ecg", "vision": "cxr"}
+    MAX_LENGTH: int = 120  # longer length to accomodate longer reports
+    TEXT_LENGTH: int = 100  # 100 words
+
+    def get_first_n_words(self, text: str, n: int = 100) -> str:
+        """97.5 percentile of text is less than 35 words"""
+        words = text.split()  # Split the text into words
+        return " ".join(words[:n])  # Join the first n words back into a string
+
+    def createCaption(self, caption: str, modality: str = "") -> str:
+        assert modality in set(self.MODALITY_LIST.keys()) or modality == "", (
+            f"modality should be in {self.MODALITY_LIST} or empty"
+        )
+        return f"text : {caption}, {modality} looks like : "
+
+    def createTokenizedCaption(self, caption: str, tokenizer: PreTrainedTokenizer) -> BatchEncoding:
+        encoding = tokenizer(
+            caption,
+            padding="max_length",
+            truncation=True,
+            max_length=self.MAX_LENGTH,
+            return_tensors="pt",
+        )
+        return encoding
+
+    def construct_caption(
+        self, caption: str, tokenizer: PreTrainedTokenizer, modality: str = ""
+    ) -> BatchEncoding:
+        """given caption string, return tokenized caption dict for model input
+        Output: dict with keys 'input_ids' and 'attention_mask', each of shape (1, L)
+        """
+        caption_str = self.createCaption(caption, modality)
+        tokenized = self.createTokenizedCaption(caption_str, tokenizer)
+        return tokenized

preprocessor.py

@@ -0,0 +1,69 @@
+import torch
+from transformers import AutoTokenizer, BatchEncoding
+
+from mixinhelpers import CXR_Mixin, ECG_Mixin, ECHO_Mixin, Text_Mixin
+
+"""
+Preprocessor classes for different modalities and their combinations.
+You can combine different mixins to create preprocessors for multi-modal inputs.
+Examples below are provided for ECHO+Text, ECG+Text, and CXR+Text.
+"""
+
+
+class BasePreprocessor:
+    def __init__(self, model_name: str = "dmis-lab/biobert-v1.1") -> None:
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+
+# duo modality preprocessors
+class ECHOText_Preprocessor(BasePreprocessor, ECHO_Mixin, Text_Mixin):
+    def __init__(self, model_name: str = "dmis-lab/biobert-v1.1") -> None:
+        super().__init__(model_name=model_name)
+
+    def preprocess_echo_text(self, echo_path: str, text: str) -> tuple[torch.Tensor, BatchEncoding]:
+        """this can be used in dataloader to correctly collate batches, use the string keys to
+        identify the modalities
+        echo_path: path to echo npy file
+        text: string of text report
+        returns: (echo tensor, tokenized text dict)"""
+        echo = self.preprocess_single_echo(echo_path)  # (C, H, W)
+        text_inputs = self.construct_caption(
+            caption=text, tokenizer=self.tokenizer, modality=self.ECHO_KEY
+        )
+        return echo, text_inputs
+
+
+class ECGText_Preprocessor(BasePreprocessor, ECG_Mixin, Text_Mixin):
+    def __init__(self, model_name: str = "dmis-lab/biobert-v1.1") -> None:
+        super().__init__(model_name=model_name)
+
+    def preprocess_ecg_text(self, ecg_path: str, text: str) -> tuple[torch.Tensor, BatchEncoding]:
+        """this can be used in dataloader to correctly collate batches, use the string keys
+        to identify the modalities
+        ecg_path: path to ecg npy file
+        text: string of text report
+        returns: (ecg tensor,  tokenized text dict)"""
+        ecg = self.preprocess_single_ecg(ecg_path)  # (C, L)
+        text_inputs = self.construct_caption(
+            caption=text, tokenizer=self.tokenizer, modality=self.ECG_KEY
+        )
+
+        return ecg, text_inputs
+
+
+class CXRText_Preprocessor(BasePreprocessor, CXR_Mixin, Text_Mixin):
+    def __init__(self, model_name: str = "dmis-lab/biobert-v1.1") -> None:
+        super().__init__(model_name=model_name)
+
+    def preprocess_cxr_text(self, cxr_path: str, text: str) -> tuple[torch.Tensor, BatchEncoding]:
+        """this can be used in dataloader to correctly collate batches, use the string keys to
+        identify the modalities
+        cxr_path: path to cxr image file
+        text: string of text report
+        returns: (cxr tensor, tokenized text dict)"""
+        cxr = self.preprocess_single_cxr(cxr_path)  # (C, H, W)
+        text_inputs = self.construct_caption(
+            caption=text, tokenizer=self.tokenizer, modality=self.VISION_KEY
+        )
+
+        return cxr, text_inputs

probmed_weights.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e706413f29b3dd7ba3ef197e1178192f92ffc2b75a0a3bd38eff37ce5f7059b0
+size 1008833750