diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..977420c449b029ba87a1ad95b8c7c59b34b13f96 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_381_4899e0da4615e883ad13.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_381_492597073098578f0605.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_381_67d9409a306e3614ec3f.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_381_7c47ba927ac118ffaacc.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_381_89ad32d31f3e70178cc1.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_381_cd644667186ae0518a3c.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_381_f7405ef7b645b3265477.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_762_09db3d4c95c46a3bae02.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_762_3349ee3cb71e3d789f00.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_762_63964662df5bc7731b7a.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_762_7ab4dededd2d3a052def.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_762_b8be5a72ea66619af007.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_762_d1c21fd700287f9f3109.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech[[:space:]]samples/eval_762_e8feadb7228a5b136ce8.wav filter=lfs diff=lfs merge=lfs -text +wandb/run-20240513_205249-qaoje1x9/run-qaoje1x9.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/finetuning_concatenated_config.json b/finetuning_concatenated_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0ebf396fec3000a411a8710d1d6c4b932ca9ff00 --- /dev/null +++ b/finetuning_concatenated_config.json @@ -0,0 +1,54 @@ +{ + "model_name_or_path": "parler-tts/parler_tts_mini_v0.1", + "feature_extractor_name": "parler-tts/dac_44khZ_8kbps", + "description_tokenizer_name": "parler-tts/parler_tts_mini_v0.1", + "prompt_tokenizer_name": "parler-tts/parler_tts_mini_v0.1", + "report_to": ["wandb"], + "overwrite_output_dir": true, + "train_dataset_name": "sanchit-gandhi/expresso-concatenated-half-normal+reach-vb/jenny_tts_dataset+sanchit-gandhi/libritts_r_test+sanchit-gandhi/libritts_r_test", + "train_metadata_dataset_name": "sanchit-gandhi/expresso-concatenated-half-normal-tags-mistral+ylacombe/jenny-tts-10k-tagged+parler-tts/libritts_r_tags_tagged_10k_generated+parler-tts/libritts_r_tags_tagged_10k_generated", + "train_dataset_config_name": "read+default+clean+other", + "train_split_name": "train+train[:20%]+test.clean+test.other", + "eval_dataset_name": "sanchit-gandhi/expresso-concatenated-half-normal+reach-vb/jenny_tts_dataset+sanchit-gandhi/libritts_r_test+sanchit-gandhi/libritts_r_test", + "eval_metadata_dataset_name": "sanchit-gandhi/expresso-concatenated-half-normal-tags-mistral+ylacombe/jenny-tts-10k-tagged+parler-tts/libritts_r_tags_tagged_10k_generated+parler-tts/libritts_r_tags_tagged_10k_generated", + "eval_dataset_config_name": "read+default+clean+other", + "eval_split_name": "train+train[:20%]+test.clean+test.other", + "max_eval_samples": 8, + "per_device_eval_batch_size": 16, + "target_audio_column_name": "audio", + "description_column_name": "text_description", + "prompt_column_name": "text", + "max_duration_in_seconds": 30.0, + "min_duration_in_seconds": 2.0, + "max_text_length": 400, + "preprocessing_num_workers": 2, + "do_train": true, + "num_train_epochs": 8, + "max_steps": -1, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "per_device_train_batch_size": 16, + "learning_rate": 0.00008, + "adam_beta1": 0.9, + "adam_beta2": 0.99, + "weight_decay": 0.01, + "lr_scheduler_type": "cosine", + "warmup_steps": 250, + "logging_steps": 5, + "freeze_text_encoder": true, + "audio_encoder_per_device_batch_size": 4, + "dtype": "bfloat16", + "seed": 456, + "output_dir": "../output_dir_training_constant_concat/", + "temporary_save_to_disk": "../audio_code_tmp_constant_concat/", + "save_to_disk": "../tmp_dataset_audio_constant_concat/", + "dataloader_num_workers": 4, + "do_eval": true, + "predict_with_generate": true, + "include_inputs_for_metrics": true, + "save_strategy": "epoch", + "evaluation_strategy": "epoch", + "save_total_limit": 5, + "group_by_length": true +} + diff --git a/run_parler_tts_training.py b/run_parler_tts_training.py new file mode 100644 index 0000000000000000000000000000000000000000..1dabcbd489f3623e49e4f5fc2d823c358aaad005 --- /dev/null +++ b/run_parler_tts_training.py @@ -0,0 +1,1763 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Train Parler-TTS using 🤗 Accelerate""" + +import logging +import os +import re +import shutil +import sys +import time +from dataclasses import dataclass, field +from datetime import timedelta +from pathlib import Path +from typing import Dict, List, Optional, Set, Union + +import datasets +import evaluate +import numpy as np +import torch +import transformers +from accelerate import Accelerator +from accelerate.utils import AutocastKwargs, InitProcessGroupKwargs, TorchDynamoPlugin, set_seed +from accelerate.utils.memory import release_memory +from datasets import Dataset, DatasetDict, IterableDataset, concatenate_datasets, interleave_datasets, load_dataset +from huggingface_hub import Repository, create_repo +from multiprocess import set_start_method +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import ( + AutoFeatureExtractor, + AutoModel, + AutoProcessor, + AutoTokenizer, + HfArgumentParser, + Seq2SeqTrainingArguments, + pipeline, +) +from transformers.optimization import get_scheduler +from transformers.trainer_pt_utils import LengthGroupedSampler +from transformers.utils import send_example_telemetry +from wandb import Audio + +from parler_tts import ( + ParlerTTSConfig, + ParlerTTSForConditionalGeneration, + build_delay_pattern_mask, +) + + +logger = logging.getLogger(__name__) + + +def list_field(default=None, metadata=None): + return field(default_factory=lambda: default, metadata=metadata) + + +_RE_CHECKPOINT = re.compile(r"^checkpoint-(\d+)-epoch-(\d+)$") + + +def get_last_checkpoint(folder): + content = os.listdir(folder) + checkpoints = [ + path + for path in content + if _RE_CHECKPOINT.search(path) is not None and os.path.isdir(os.path.join(folder, path)) + ] + if len(checkpoints) == 0: + return + return os.path.join(folder, max(checkpoints, key=lambda x: int(_RE_CHECKPOINT.search(x).groups()[0]))) + + +def sorted_checkpoints(output_dir=None, checkpoint_prefix="checkpoint") -> List[str]: + """Helper function to sort saved checkpoints from oldest to newest.""" + ordering_and_checkpoint_path = [] + + glob_checkpoints = [str(x) for x in Path(output_dir).glob(f"{checkpoint_prefix}-*") if os.path.isdir(x)] + + for path in glob_checkpoints: + regex_match = re.match(f".*{checkpoint_prefix}-([0-9]+)", path) + if regex_match is not None and regex_match.groups() is not None: + ordering_and_checkpoint_path.append((int(regex_match.groups()[0]), path)) + + checkpoints_sorted = sorted(ordering_and_checkpoint_path) + checkpoints_sorted = [checkpoint[1] for checkpoint in checkpoints_sorted] + return checkpoints_sorted + + +def rotate_checkpoints(save_total_limit=None, output_dir=None, checkpoint_prefix="checkpoint") -> None: + """Helper function to delete old checkpoints.""" + if save_total_limit is None or save_total_limit <= 0: + return + # Check if we should delete older checkpoint(s) + checkpoints_sorted = sorted_checkpoints(output_dir=output_dir, checkpoint_prefix=checkpoint_prefix) + if len(checkpoints_sorted) <= save_total_limit: + return + + number_of_checkpoints_to_delete = max(0, len(checkpoints_sorted) - save_total_limit) + checkpoints_to_be_deleted = checkpoints_sorted[:number_of_checkpoints_to_delete] + for checkpoint in checkpoints_to_be_deleted: + logger.info(f"Deleting older checkpoint [{checkpoint}] due to args.save_total_limit") + shutil.rmtree(checkpoint, ignore_errors=True) + + +def log_metric( + accelerator, + metrics: Dict, + train_time: float, + step: int, + epoch: int, + learning_rate: float = None, + prefix: str = "train", +): + """Helper function to log all training/evaluation metrics with the correct prefixes and styling.""" + log_metrics = {} + for k, v in metrics.items(): + log_metrics[f"{prefix}/{k}"] = v + log_metrics[f"{prefix}/time"] = train_time + log_metrics[f"{prefix}/epoch"] = epoch + if learning_rate is not None: + log_metrics[f"{prefix}/learning_rate"] = learning_rate + accelerator.log(log_metrics, step=step) + + +def log_pred( + accelerator, + pred_descriptions: List[str], + pred_prompts: List[str], + transcriptions: List[str], + audios: List[torch.Tensor], + sampling_rate: int, + step: int, + prefix: str = "eval", + num_lines: int = 200000, +): + """Helper function to log target/predicted transcriptions to weights and biases (wandb).""" + if accelerator.is_main_process: + wandb_tracker = accelerator.get_tracker("wandb") + # pretty name for current step: step 50000 -> step 50k + cur_step_pretty = f"{int(step // 1000)}k" if step > 1000 else step + prefix_pretty = prefix.replace("/", "-") + + # convert str data to a wandb compatible format + str_data = [[pred_descriptions[i], pred_prompts[i], transcriptions[i]] for i in range(len(pred_descriptions))] + # log as a table with the appropriate headers + wandb_tracker.log_table( + table_name=f"predictions/{prefix_pretty}-step-{cur_step_pretty}", + columns=["Target descriptions", "Target prompts", "Predicted transcriptions"], + data=str_data[:num_lines], + step=step, + commit=False, + ) + + # wandb can only loads 100 audios per step + wandb_tracker.log( + { + f"Speech samples/{prefix}": [ + Audio( + audio, + caption=f"{pred_prompts[i]} --- DESCRIPTION: {pred_descriptions[i]}", + sample_rate=sampling_rate, + ) + for (i, audio) in enumerate(audios[: min(len(audios), 100)]) + ] + }, + step=step, + ) + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune from. + """ + + model_name_or_path: str = field( + metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} + ) + config_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} + ) + feature_extractor_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained feature extractor name or path if not the same as model_name"} + ) + description_tokenizer_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained description tokenizer name or path if not the same as model_name"} + ) + prompt_tokenizer_name: Optional[str] = field( + default=None, + metadata={"help": "Pretrained prompt tokenizer name or path if not the same as description_tokenizer_name"}, + ) + cache_dir: Optional[str] = field( + default=None, + metadata={"help": "Where to store the pretrained models downloaded from huggingface.co"}, + ) + use_fast_tokenizer: bool = field( + default=True, + metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, + ) + model_revision: str = field( + default="main", + metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."}, + ) + pad_token_id: int = field( + default=None, + metadata={"help": "If specified, change the model pad token id."}, + ) + decoder_start_token_id: int = field( + default=None, + metadata={"help": "If specified, change the model decoder start token id."}, + ) + freeze_text_encoder: bool = field( + default=False, + metadata={"help": "Whether to freeze the text encoder."}, + ) + do_sample: bool = field( + default=True, + metadata={"help": "Whether to do sampling or greedy decoding."}, + ) + temperature: float = field( + default=1.0, + metadata={"help": "Temperature if sampling."}, + ) + max_length: int = field( + default=2580, + metadata={"help": "Generation max length."}, + ) + bandwidth: float = field( + default=6, + metadata={"help": "Audio encoder bandwidth."}, + ) + asr_model_name_or_path: str = field( + default="distil-whisper/distil-large-v2", + metadata={ + "help": "Used to compute WER during evaluation. Path to pretrained model or model identifier from huggingface.co/models" + }, + ) + clap_model_name_or_path: str = field( + default="laion/larger_clap_music_and_speech", + metadata={ + "help": "Used to compute audio similarity during evaluation. Path to pretrained model or model identifier from huggingface.co/models" + }, + ) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + + Using `HfArgumentParser` we can turn this class + into argparse arguments to be able to specify them on + the command line. + """ + + train_dataset_name: str = field( + default=None, + metadata={ + "help": "The name of the training dataset to use (via the datasets library). Load and combine " + "multiple datasets by separating dataset ids by a '+' symbol. For example, to load and combine " + " librispeech and common voice, set `train_dataset_name='librispeech_asr+common_voice'`." + }, + ) + train_dataset_config_name: Optional[str] = field( + default=None, + metadata={ + "help": "The configuration name of the training dataset to use (via the datasets library). Load and combine " + "multiple datasets by separating dataset configs by a '+' symbol." + }, + ) + train_split_name: str = field( + default="train", + metadata={ + "help": ("The name of the training data set split to use (via the datasets library). Defaults to 'train'") + }, + ) + train_dataset_samples: str = field( + default=None, + metadata={ + "help": "Number of samples in the training data. Load and combine " + "multiple datasets by separating dataset samples by a '+' symbol." + }, + ) + train_metadata_dataset_name: str = field( + default=None, + metadata={ + "help": "The name of the metadata training dataset to use (via the datasets library). Load and combine " + "multiple datasets by separating dataset ids by a '+' symbol. For example, to load and combine " + " librispeech and common voice, set `train_dataset_name='librispeech_asr+common_voice'`." + }, + ) + eval_dataset_name: str = field( + default=None, + metadata={ + "help": "The name of the evaluation dataset to use (via the datasets library). Defaults to the training dataset name if unspecified." + }, + ) + eval_dataset_config_name: Optional[str] = field( + default=None, + metadata={ + "help": "The configuration name of the evaluation dataset to use (via the datasets library). Defaults to the training dataset config name if unspecified" + }, + ) + eval_split_name: str = field( + default="test", + metadata={ + "help": "The name of the evaluation data set split to use (via the datasets library). Defaults to 'test'" + }, + ) + eval_metadata_dataset_name: str = field( + default=None, + metadata={ + "help": "The name of the metadata training dataset to use (via the datasets library). Load and combine " + "multiple datasets by separating dataset ids by a '+' symbol. For example, to load and combine " + " librispeech and common voice, set `train_dataset_name='librispeech_asr+common_voice'`." + }, + ) + target_audio_column_name: str = field( + default="audio", + metadata={"help": "The name of the dataset column containing the target audio data. Defaults to 'audio'"}, + ) + description_column_name: str = field( + default=None, + metadata={"help": "The name of the dataset column containing the description text data. Defaults to 'None'."}, + ) + prompt_column_name: str = field( + default=None, + metadata={"help": "The name of the dataset column containing the prompt text data. Defaults to 'None'."}, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."} + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + max_train_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ) + }, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of validation examples to this " + "value if set." + ) + }, + ) + max_duration_in_seconds: float = field( + default=35.0, + metadata={ + "help": ( + "Filter audio files that are longer than `max_duration_in_seconds` seconds to 'max_duration_in_seconds`." + "Also, used to set maximum audio length if `pad_to_max_length=True`." + ) + }, + ) + min_duration_in_seconds: float = field( + default=0.0, metadata={"help": "Filter audio files that are shorter than `min_duration_in_seconds` seconds"} + ) + max_text_length: int = field( + default=500, metadata={"help": "If set, max description lengths in number of characters."} + ) + max_prompt_token_length: int = field( + default=None, + metadata={ + "help": ( + "If set, filter samples with prompts that are longer than `max_prompt_token_length` tokens." + "Also, used to set maximum prompt token length if `pad_to_max_length=True`." + ) + }, + ) + max_description_token_length: int = field( + default=None, + metadata={ + "help": ( + "If set, filter samples with descriptions that are longer than `max_description_token_length` tokens." + "Also, used to set maximum desription token length if `pad_to_max_length=True`." + ) + }, + ) + pad_to_max_length: bool = field( + default=False, + metadata={ + "help": ( + "If `True`, pad audio, prompt and description to a maximum length set with respectively " + "`max_duration_in_seconds`, `max_prompt_token_length`, `max_description_token_length`." + ) + }, + ) + preprocessing_only: bool = field( + default=False, + metadata={ + "help": ( + "Whether to only do data preprocessing and skip training. This is especially useful when data" + " preprocessing errors out in distributed training due to timeout. In this case, one should run the" + " preprocessing in a non-distributed setup with `preprocessing_only=True` so that the cached datasets" + " can consequently be loaded in distributed training." + " In this training script, `save_to_disk` must be set to the path in which the dataset should be saved. " + ) + }, + ) + token: str = field( + default=None, + metadata={ + "help": ( + "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token " + "generated when running `huggingface-cli login` (stored in `~/.huggingface`)." + ) + }, + ) + use_auth_token: bool = field( + default=None, + metadata={ + "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token` instead." + }, + ) + trust_remote_code: bool = field( + default=False, + metadata={ + "help": ( + "Whether or not to allow for custom models defined on the Hub in their own modeling files. This option " + "should only be set to `True` for repositories you trust and in which you have read the code, as it will " + "execute code present on the Hub on your local machine." + ) + }, + ) + add_audio_samples_to_wandb: bool = field( + default=False, + metadata={"help": "If set and if `wandb` in args.report_to, will add generated audio samples to wandb logs."}, + ) + id_column_name: str = field(default=None, metadata={"help": "id column name."}) + wandb_project: str = field( + default="parler-speech", + metadata={"help": "The name of the wandb project."}, + ) + save_to_disk: str = field( + default=None, + metadata={ + "help": "If set, will save the dataset to this path if this is an empyt folder. If not empty, will load the datasets from it." + }, + ) + temporary_save_to_disk: str = field(default=None, metadata={"help": "Temporarily save audio labels here."}) + pad_to_multiple_of: Optional[int] = field( + default=2, + metadata={"help": ("Pad to multiple of for tokenizers.")}, + ) + + +@dataclass +class ParlerTTSTrainingArguments(Seq2SeqTrainingArguments): + dtype: Optional[str] = field( + default="float32", + metadata={ + "help": ( + "The data type (dtype) in which to run training. One of `float32` (full-precision), " + "`float16` or `bfloat16` (both half-precision)." + ) + }, + ) + audio_encoder_per_device_batch_size: int = field( + default=8, + metadata={"help": ("Specify the batch size of the audio encoding pre-processing steps.")}, + ) + + +@dataclass +class DataCollatorEncodecWithPadding: + """ + Data collator that will dynamically pad the inputs received to the longest sequence in the batch or + to `max_length` if `max_length` is set and `padding=max_length`. + """ + + feature_extractor: AutoFeatureExtractor + audio_column_name: str + feature_extractor_input_name: Optional[str] = "input_values" + max_length: Optional[int] = None + padding: Optional[str] = "longest" + + def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]: + # split inputs and labels since they have to be of different lengths and need + # different padding methods + audios = [feature[self.audio_column_name]["array"] for feature in features] + len_audio = [len(audio) for audio in audios] + + batch = self.feature_extractor(audios, return_tensors="pt", padding=self.padding, max_length=self.max_length, sampling_rate=self.feature_extractor.sampling_rate) + batch["len_audio"] = torch.tensor(len_audio).unsqueeze(1) + return batch + + +@dataclass +class DataCollatorParlerTTSWithPadding: + """ + Data collator that will dynamically pad the inputs received. + Args: + prompt_tokenizer (:class:`~transformers.AutoTokenizer`) + The prompt_tokenizer used for proccessing the data. + description_tokenizer (:class:`~transformers.AutoTokenizer`) + The description_tokenizer used for proccessing the data. + padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding index) + among: + * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single + sequence if provided). + * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the + maximum acceptable input length for the model if that argument is not provided. + * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of + different lengths). + pad_to_multiple_of (:obj:`int`, `optional`): + If set will pad the sequence to a multiple of the provided value. + This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= + 7.5 (Volta). + """ + + prompt_tokenizer: AutoTokenizer + description_tokenizer: AutoTokenizer + padding: Union[bool, str] = "longest" + pad_to_multiple_of: Optional[int] = None + prompt_max_length: Optional[int] = None + description_max_length: Optional[int] = None + audio_max_length: Optional[int] = None + + def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]: + # split inputs and labels since they have to be of different lengths and need + # different padding methods + + labels = [torch.tensor(feature["labels"]).transpose(0, 1) for feature in features] + # (bsz, seq_len, num_codebooks) + labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True, padding_value=-100) + if self.audio_max_length is not None and self.padding == "max_length": + labels = torch.nn.functional.pad(labels, pad=(0, 0, 0, max(self.audio_max_length - labels.shape[1], 0))) + + input_ids = [{"input_ids": feature["input_ids"]} for feature in features] + + input_ids = self.description_tokenizer.pad( + input_ids, + return_tensors="pt", + padding=self.padding, + pad_to_multiple_of=self.pad_to_multiple_of, + max_length=self.description_max_length, + ) + + batch = {"labels": labels, **input_ids} + + if self.audio_max_length is not None and self.padding == "max_length": + # if we do torch.compile, we need to also specify the attention_mask + decoder_attention_mask = torch.ones(labels.shape[:2], dtype=input_ids["attention_mask"].dtype) + batch["decoder_attention_mask"] = decoder_attention_mask + + prompt_input_ids = [{"input_ids": feature["prompt_input_ids"]} for feature in features] + prompt_input_ids = self.prompt_tokenizer.pad( + prompt_input_ids, + return_tensors="pt", + padding=self.padding, + pad_to_multiple_of=self.pad_to_multiple_of, + max_length=self.prompt_max_length, + ) + + batch["prompt_input_ids"] = prompt_input_ids["input_ids"] + if "attention_mask" in prompt_input_ids: + batch["prompt_attention_mask"] = prompt_input_ids["attention_mask"] + + return batch + + +def convert_dataset_str_to_list( + dataset_names, + dataset_config_names, + metadata_dataset_names=None, + splits=None, + dataset_samples=None, + default_split="train", +): + if isinstance(dataset_names, str): + dataset_names = dataset_names.split("+") + dataset_config_names = dataset_config_names.split("+") + splits = splits.split("+") if splits is not None else None + dataset_samples = dataset_samples.split("+") if dataset_samples is not None else None + metadata_dataset_names = metadata_dataset_names.split("+") if metadata_dataset_names is not None else None + + # basic checks to ensure we've got the right number of datasets/configs/splits/columns/probs + if len(dataset_names) != len(dataset_config_names): + raise ValueError( + f"Ensure one config is passed for each dataset, got {len(dataset_names)} datasets and" + f" {len(dataset_config_names)} configs." + ) + + if splits is not None and len(splits) != len(dataset_names): + raise ValueError( + f"Ensure one split is passed for each dataset, got {len(dataset_names)} datasets and {len(splits)} splits." + ) + + if metadata_dataset_names is not None and len(metadata_dataset_names) != len(dataset_names): + raise ValueError( + f"Ensure one metadata dataset is passed for each dataset, got {len(dataset_names)} datasets and {len(metadata_dataset_names)} metadata datasets." + ) + + if dataset_samples is not None: + if len(dataset_samples) != len(dataset_names): + raise ValueError( + f"Ensure one sample is passed for each dataset, got {len(dataset_names)} datasets and " + f"{len(dataset_samples)} samples." + ) + dataset_samples = [float(ds_sample) for ds_sample in dataset_samples] + else: + dataset_samples = [None] * len(dataset_names) + + splits = splits if splits is not None else [default_split for _ in range(len(dataset_names))] + + dataset_names_dict = [] + for i, ds_name in enumerate(dataset_names): + dataset_names_dict.append( + { + "name": ds_name, + "config": dataset_config_names[i], + "split": splits[i], + "metadata_dataset_name": metadata_dataset_names[i], + "samples": dataset_samples[i], + } + ) + return dataset_names_dict + + +def load_multiple_datasets( + accelerator: Accelerator, + dataset_names: Union[List, str], + dataset_config_names: Union[List, str], + metadata_dataset_names: Optional[str] = None, + splits: Optional[Union[List, str]] = None, + label_column_names: Optional[List] = None, + stopping_strategy: Optional[str] = "first_exhausted", + dataset_samples: Optional[Union[List, np.array]] = None, + streaming: Optional[bool] = False, + seed: Optional[int] = None, + id_column_name: Optional[str] = None, + columns_to_keep: Optional[Set[str]] = None, + prompt_column_name: Optional[str] = None, + sampling_rate: Optional[int] = None, + audio_column_name: Optional[str] = None, + **kwargs, +) -> Union[Dataset, IterableDataset]: + dataset_names_dict = convert_dataset_str_to_list( + dataset_names, dataset_config_names, metadata_dataset_names, splits, label_column_names, dataset_samples + ) + + if dataset_samples is not None: + dataset_samples = [ds_dict["samples"] for ds_dict in dataset_names_dict] + probabilities = np.array(dataset_samples) / np.sum(dataset_samples) + else: + probabilities = None + + all_datasets = [] + # iterate over the datasets we want to interleave + for dataset_dict in tqdm(dataset_names_dict, desc="Combining datasets..."): + with accelerator.main_process_first(): + dataset = load_dataset( + dataset_dict["name"], + dataset_dict["config"], + split=dataset_dict["split"], + streaming=streaming, + **kwargs, + ) + dataset_features = dataset.features.keys() + + if sampling_rate is not None and audio_column_name is not None: + # resample target audio + dataset = dataset.cast_column(audio_column_name, datasets.features.Audio(sampling_rate=sampling_rate)) + + metadata_dataset_name = dataset_dict["metadata_dataset_name"] + if metadata_dataset_name is not None: + logger.info( + f'Merging {dataset_dict["name"]} - {dataset_dict["split"]} with {metadata_dataset_name} - {dataset_dict["split"]}' + ) + metadata_dataset = load_dataset( + metadata_dataset_name, + dataset_dict["config"], + split=dataset_dict["split"], + streaming=streaming, + **kwargs, + ) + + # TODO(YL): I forgot to create unique ids for MLS english. + # To iterate faster, I bypass the original id check and do another one. - Done once because assuming it won't change next time + # if dataset_dict["name"] == "parler-tts/mls_eng_10k": + # def concat_ids(book_id, speaker_id, begin_time): + # return {"id": f"{book_id}_{speaker_id}_{str(begin_time).replace('.', '_')}"} + # dataset = dataset.map(concat_ids, input_columns=["book_id", "speaker_id", "begin_time"], num_proc=24) + # metadata_dataset = metadata_dataset.map(concat_ids, input_columns=["book_id", "speaker_id", "begin_time"], num_proc=24) + # metadata_dataset = metadata_dataset.rename_column(id_column_name, f"metadata_{id_column_name}") + + if dataset_dict["name"] != "parler-tts/mls_eng_10k": + if id_column_name is not None and id_column_name not in dataset.column_names: + raise ValueError( + f"id_column_name={id_column_name} but has not been found in the dataset columns" + f"- one of {', '.join(list(dataset.column_names))}." + ) + if id_column_name is not None and id_column_name not in metadata_dataset.column_names: + raise ValueError( + f"id_column_name={id_column_name} but has not been found in the metadata dataset columns" + f"- one of {', '.join(list(metadata_dataset.column_names))}." + ) + elif id_column_name is not None: + metadata_dataset = metadata_dataset.rename_column(id_column_name, f"metadata_{id_column_name}") + + metadata_columns_to_remove = set(metadata_dataset.column_names).intersection(set(dataset.column_names)) + + if prompt_column_name is not None: + # We might have applied some transformations to the prompts (e.g punctuation restoration) + # so we make sure to remove it from the original dataset + if prompt_column_name in dataset.column_names: + logger.info( + f"REMOVE {prompt_column_name} from dataset {dataset_dict['name']} - dataset_dict['split']" + ) + dataset.remove_columns(prompt_column_name) + + metadata_columns_to_remove = set(metadata_dataset.column_names).intersection(set(dataset.column_names)) + metadata_dataset = metadata_dataset.remove_columns(metadata_columns_to_remove) + + dataset = concatenate_datasets([dataset, metadata_dataset], axis=1) + + if id_column_name is not None and dataset_dict["name"] != "parler-tts/mls_eng_10k": + if ( + len( + dataset.filter( + lambda id1, id2: id1 != id2, + input_columns=[id_column_name, f"metadata_{id_column_name}"], + ) + ) + != 0 + ): + raise ValueError( + f"Concatenate didn't work. Some ids don't correspond on dataset {dataset_dict['name']}" + ) + + dataset_features = dataset.features.keys() + + if columns_to_keep is not None: + dataset = dataset.remove_columns(set(dataset_features - columns_to_keep)) + all_datasets.append(dataset) + + if len(all_datasets) == 1: + # we have a single dataset so just return it as is + return all_datasets[0] + + if streaming: + interleaved_dataset = interleave_datasets( + all_datasets, + stopping_strategy=stopping_strategy, + probabilities=probabilities, + seed=seed, + ) + else: + with accelerator.main_process_first(): + interleaved_dataset = concatenate_datasets(all_datasets) + + return interleaved_dataset + + +def main(): + # See all possible arguments in src/transformers/training_args.py + # or by passing the --help flag to this script. + # We now keep distinct sets of args, for a cleaner separation of concerns. + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, ParlerTTSTrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The + # information sent is the one passed as arguments along with your Python/PyTorch versions. + send_example_telemetry("run_parler_tts", model_args, data_args) + + if training_args.dtype == "float16": + mixed_precision = "fp16" + elif training_args.dtype == "bfloat16": + mixed_precision = "bf16" + else: + mixed_precision = "no" + + if data_args.pad_to_max_length and ( + data_args.max_duration_in_seconds is None + or data_args.max_prompt_token_length is None + or data_args.max_description_token_length is None + ): + raise ValueError( + "`pad_to_max_length` is `True` but one of the following parameters has not been set: `max_duration_in_seconds`, `max_prompt_token_length`, `max_description_token_length`" + ) + + padding = "max_length" if data_args.pad_to_max_length else "longest" + + ####### A. Preparation + kwargs_handlers = [InitProcessGroupKwargs(timeout=timedelta(minutes=60))] + if training_args.torch_compile: + # TODO(YL): add more compile modes? + kwargs_handlers.append(TorchDynamoPlugin(backend="inductor", mode="default")) # reduce-overhead + + accelerator = Accelerator( + gradient_accumulation_steps=training_args.gradient_accumulation_steps, + mixed_precision=mixed_precision, + log_with=training_args.report_to, + project_dir=training_args.output_dir, + kwargs_handlers=kwargs_handlers, + ) + + accelerator.init_trackers( + project_name=data_args.wandb_project, + config={ + "learning_rate": training_args.learning_rate, + "model_name_or_path": model_args.model_name_or_path, + "num_train_epochs": training_args.num_train_epochs, + "gradient_accumulation_steps": training_args.gradient_accumulation_steps, + "per_device_train_batch_size": training_args.per_device_train_batch_size, + "global_batch_size": training_args.per_device_train_batch_size * accelerator.num_processes, + "mixed_precision": mixed_precision, + "lr_scheduler_type": training_args.lr_scheduler_type, + "warmup_steps": training_args.warmup_steps, + "freeze_text_encoder": model_args.freeze_text_encoder, + "max_duration_in_seconds": data_args.max_duration_in_seconds, + "weight_decay": training_args.weight_decay, + "adam_beta1": training_args.adam_beta1, + "adam_beta2": training_args.adam_beta2, + "temperature": model_args.temperature, + }, + ) + + # Detecting last checkpoint and eventually continue from last checkpoint + last_checkpoint = None + if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty. " + "Use --overwrite_output_dir to overcome." + ) + elif last_checkpoint is not None and training_args.resume_from_checkpoint is None: + logger.info( + f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " + "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." + ) + + # Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + logger.setLevel(logging.INFO if accelerator.is_main_process else logging.WARN) + + # Log a small summary on each proces + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, " + f"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}" + ) + + # Set the verbosity to info of the Transformers logger (on main process only) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + + logger.info("Training/evaluation parameters %s", training_args) + + # Set seed before initializing model. + set_seed(training_args.seed) + num_workers = data_args.preprocessing_num_workers + + # 1. First, lett's instantiate the feature extractor, tokenizers and model + # Note for distributed training, the .from_pretrained methods guarantee that only + # one local process can concurrently download model & vocab. + + # load feature extractor + feature_extractor = AutoFeatureExtractor.from_pretrained( + model_args.feature_extractor_name or model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + token=data_args.token, + trust_remote_code=data_args.trust_remote_code, + ) + sampling_rate = feature_extractor.sampling_rate + + # load prompt tokenizer + prompt_tokenizer = AutoTokenizer.from_pretrained( + model_args.prompt_tokenizer_name or model_args.description_tokenizer_name or model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + token=data_args.token, + trust_remote_code=data_args.trust_remote_code, + use_fast=model_args.use_fast_tokenizer, + padding_side="left", # prompt has to be padded on the left bc it's preprend to codebooks hidden states + ) + + # load description tokenizer + description_tokenizer = AutoTokenizer.from_pretrained( + model_args.description_tokenizer_name or model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + token=data_args.token, + trust_remote_code=data_args.trust_remote_code, + use_fast=model_args.use_fast_tokenizer, + ) + + if model_args.use_fast_tokenizer: + logger.warning( + "Disabling fast tokenizer warning: https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py#L3231-L3235" + ) + prompt_tokenizer.deprecation_warnings["Asking-to-pad-a-fast-tokenizer"] = True + description_tokenizer.deprecation_warnings["Asking-to-pad-a-fast-tokenizer"] = True + + # 2. Now, let's load the dataset + + if data_args.save_to_disk is not None: + os.makedirs(data_args.save_to_disk, exist_ok=True) + + # assume that the dataset has been saved to `save_to_disk` if the latter is not empty + dataset_was_precomputed = len(os.listdir(data_args.save_to_disk)) > 0 + if dataset_was_precomputed: + vectorized_datasets = datasets.load_from_disk(data_args.save_to_disk) + else: + raw_datasets = DatasetDict() + + columns_to_keep = { + "target_audio_column_name": data_args.target_audio_column_name, + "prompt_column_name": data_args.prompt_column_name, + } + if data_args.description_column_name is not None: + columns_to_keep["description_column_name"] = data_args.description_column_name + + if training_args.do_train: + raw_datasets["train"] = load_multiple_datasets( + accelerator, + data_args.train_dataset_name, + data_args.train_dataset_config_name, + metadata_dataset_names=data_args.train_metadata_dataset_name, + splits=data_args.train_split_name, + dataset_samples=data_args.train_dataset_samples, + seed=training_args.seed, + cache_dir=model_args.cache_dir, + num_proc=data_args.preprocessing_num_workers, + id_column_name=data_args.id_column_name, + columns_to_keep=columns_to_keep.values(), + prompt_column_name=data_args.prompt_column_name, + audio_column_name=data_args.target_audio_column_name, + sampling_rate=sampling_rate, + # streaming=data_args.streaming, TODO(SG): optionally enable streaming mode + ) + + for key in columns_to_keep: + if columns_to_keep[key] not in raw_datasets["train"].column_names: + raise ValueError( + f"--{key} '{columns_to_keep[key]}' not found in dataset '{data_args.train_dataset_name}'." + f" Make sure to set `--{key}` to the correct audio column - one of" + f" {', '.join(raw_datasets['train'].column_names)}." + ) + + if data_args.max_train_samples is not None: + raw_datasets["train"] = raw_datasets["train"].select(range(data_args.max_train_samples)) + + if training_args.do_eval: + raw_datasets["eval"] = load_multiple_datasets( + accelerator, + data_args.eval_dataset_name if data_args.eval_dataset_name else data_args.train_dataset_name, + data_args.eval_dataset_config_name + if data_args.eval_dataset_config_name + else data_args.train_dataset_config_name, + metadata_dataset_names=data_args.eval_metadata_dataset_name, + splits=data_args.eval_split_name, + cache_dir=model_args.cache_dir, + num_proc=data_args.preprocessing_num_workers, + id_column_name=data_args.id_column_name, + columns_to_keep=columns_to_keep.values(), + prompt_column_name=data_args.prompt_column_name, + audio_column_name=data_args.target_audio_column_name, + sampling_rate=sampling_rate, + # streaming=data_args.streaming, TODO(SG): optionally enable streaming mode + ) + + if data_args.max_eval_samples is not None: + raw_datasets["eval"] = ( + raw_datasets["eval"].shuffle(seed=training_args.seed).select(range(data_args.max_eval_samples)) + ) + + # 3. Next, let's load the config. + config = ParlerTTSConfig.from_pretrained( + model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + token=data_args.token, + trust_remote_code=data_args.trust_remote_code, + ) + + # update pad token id and decoder_start_token_id + config.update( + { + "pad_token_id": model_args.pad_token_id if model_args.pad_token_id is not None else config.pad_token_id, + "decoder_start_token_id": ( + model_args.decoder_start_token_id + if model_args.decoder_start_token_id is not None + else config.decoder_start_token_id + ), + } + ) + + # create model + model = ParlerTTSForConditionalGeneration.from_pretrained( + model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + config=config, + token=data_args.token, + trust_remote_code=data_args.trust_remote_code, + ) + + # enable gradient checkpointing if necessary + if training_args.gradient_checkpointing: + model.gradient_checkpointing_enable() + + # 4. Now we preprocess the datasets including loading the audio, resampling and normalization + # Thankfully, `datasets` takes care of automatically loading and resampling the audio, + # so that we just need to set the correct target sampling rate and normalize the input + # via the `feature_extractor` + + # derive max & min input length for sample rate & max duration + sampling_rate = feature_extractor.sampling_rate + max_target_length = data_args.max_duration_in_seconds * sampling_rate + min_target_length = data_args.min_duration_in_seconds * sampling_rate + target_audio_column_name = data_args.target_audio_column_name + description_column_name = data_args.description_column_name + prompt_column_name = data_args.prompt_column_name + feature_extractor_input_name = feature_extractor.model_input_names[0] + audio_encoder_pad_token_id = config.decoder.pad_token_id + audio_encoder_eos_token_id = config.decoder.eos_token_id + audio_encoder_bos_token_id = model.generation_config.decoder_start_token_id + max_length = model.generation_config.max_length + num_codebooks = model.decoder.config.num_codebooks + bandwidth = model_args.bandwidth + + # Freeze Encoders + model.freeze_encoders(model_args.freeze_text_encoder) + + # Test all gather - used for warmout and avoiding timeout + test_tensor = torch.tensor([accelerator.process_index], device=accelerator.device) + gathered_tensor = accelerator.gather(test_tensor) + print("gathered_tensor", gathered_tensor) + accelerator.wait_for_everyone() + + if not dataset_was_precomputed: + # Filter on text length + if description_column_name is not None and data_args.max_text_length is not None: + with accelerator.main_process_first(): + # filter description that is shorter than max_text_length + raw_datasets = raw_datasets.filter( + lambda x: len(x) < data_args.max_text_length, + num_proc=num_workers, + input_columns=[description_column_name], + ) + + # Preprocessing the dataset. + # We need to tokenize the texts. + def pass_through_processors(description, prompt): + batch = {} + + batch["input_ids"] = description_tokenizer(description.strip())["input_ids"] + batch["prompt_input_ids"] = prompt_tokenizer(prompt.strip())["input_ids"] + + return batch + + with accelerator.main_process_first(): + # this is a trick to avoid to rewrite the entire audio column which takes ages + vectorized_datasets = raw_datasets.map( + pass_through_processors, + remove_columns=next(iter(raw_datasets.values())).column_names, + input_columns=[description_column_name, prompt_column_name], + num_proc=num_workers, + desc="preprocess datasets", + ) + + # We use Accelerate to perform distributed inference + # T5 doesn't support fp16 + autocast_kwargs = AutocastKwargs(enabled=(mixed_precision != "fp16")) + + # Now we encode the audio labels with encodec. + ####### B. Encode audio + + logger.info("*** Encode target audio with encodec ***") + + # no need to prepare audio_decoder because used for inference without mixed precision + # see: https://huggingface.co/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.prepare + if training_args.torch_compile: + audio_decoder = accelerator.prepare_model(model.audio_encoder, evaluation_mode=True) + else: + audio_decoder = model.audio_encoder + + encoder_data_collator = DataCollatorEncodecWithPadding( + feature_extractor, + audio_column_name=target_audio_column_name, + feature_extractor_input_name=feature_extractor_input_name, + max_length=max_target_length, + padding=padding, + ) + + def apply_audio_decoder(batch): + len_audio = batch.pop("len_audio") + audio_decoder.to(batch["input_values"].device).eval() + with torch.no_grad(): + labels = audio_decoder.encode(**batch, bandwidth=bandwidth)["audio_codes"] + output = {} + output["len_audio"] = len_audio + # (1, bsz, codebooks, seq_len) -> (bsz, seq_len, codebooks) + output["labels"] = labels.squeeze(0).transpose(1, 2) + output["ratio"] = torch.ones_like(len_audio) * labels.shape[-1] / len_audio.max() + return output + + for split in vectorized_datasets: + data_loader = DataLoader( + raw_datasets[split], + batch_size=training_args.audio_encoder_per_device_batch_size, + collate_fn=encoder_data_collator, + num_workers=training_args.dataloader_num_workers, + pin_memory=True, + ) + data_loader = accelerator.prepare(data_loader) + + all_generated_labels = [] + all_lens = [] + for batch in tqdm(data_loader, disable=not accelerator.is_local_main_process): + generate_labels = apply_audio_decoder(batch) + generate_labels = accelerator.pad_across_processes(generate_labels, dim=1, pad_index=0) + generate_labels = accelerator.gather_for_metrics(generate_labels) + + if accelerator.is_main_process: + lab = generate_labels["labels"].cpu().transpose(1, 2).to(torch.int16) + rat = generate_labels["ratio"].cpu().squeeze() + lens = generate_labels["len_audio"].cpu().squeeze() + lab = [l[:, : int(ratio * length)] for (l, ratio, length) in zip(lab, rat, lens)] + + all_generated_labels.extend(lab) + all_lens.extend(lens) + + # (1, codebooks, seq_len) where seq_len=1 + bos_labels = torch.ones((1, num_codebooks, 1)) * audio_encoder_bos_token_id + + if accelerator.is_main_process: + tmp_labels = Dataset.from_dict({"labels": all_generated_labels, "target_length": all_lens}) + tmp_labels.save_to_disk( + os.path.join(data_args.temporary_save_to_disk, split), + num_proc=1 if split == "eval" else data_args.preprocessing_num_workers, + ) + accelerator.wait_for_everyone() + del all_generated_labels + + tmp_labels = datasets.load_from_disk(os.path.join(data_args.temporary_save_to_disk, split)) + with accelerator.main_process_first(): + vectorized_datasets[split] = concatenate_datasets([vectorized_datasets[split], tmp_labels], axis=1) + + def postprocess_dataset(labels): + # (1, codebooks, seq_len) + labels = torch.tensor(labels).unsqueeze(0) + # add bos + labels = torch.cat([bos_labels, labels], dim=-1) + + labels, delay_pattern_mask = build_delay_pattern_mask( + labels, + bos_token_id=audio_encoder_bos_token_id, + pad_token_id=audio_encoder_eos_token_id, + max_length=labels.shape[-1] + num_codebooks, + num_codebooks=num_codebooks, + ) + + # the first ids of the delay pattern mask are precisely labels, we use the rest of the labels mask + # to take care of EOS + # we want labels to look like this: + # - [B, a, b, E, E, E, E] + # - [B, B, c, d, E, E, E] + # - [B, B, B, e, f, E, E] + # - [B, B, B, B, g, h, E] + labels = torch.where(delay_pattern_mask == -1, audio_encoder_eos_token_id, delay_pattern_mask) + + # the first timestamp is associated to a row full of BOS, let's get rid of it + # we also remove the last timestampts (full of PAD) + output = {"labels": labels[:, 1:]} + return output + + with accelerator.main_process_first(): + vectorized_datasets[split] = vectorized_datasets[split].map( + postprocess_dataset, + num_proc=data_args.preprocessing_num_workers, # this one is resource consuming if many processor. + input_columns=["labels"], + desc="Postprocessing labeling", + ) + + accelerator.free_memory() + del generate_labels, all_lens + + with accelerator.main_process_first(): + # NOTE: filtering is done at the end because in the `datasets` library, caching audio files is done after most operations + # caching audio files is time and disk-space consuming, so we want to avoid it at all costs, especially for large (>1Kh) audio datasets. + # That's also why we avoid to concat the processed datasets (vectorized_datasets) with the audio column present in raw_datasets. + + def is_audio_in_length_range(length): + return length > min_target_length and length < max_target_length + + # filter data that is shorter than min_target_length + vectorized_datasets = vectorized_datasets.filter( + is_audio_in_length_range, + num_proc=num_workers, + input_columns=["target_length"], + ) + + if description_column_name is not None and data_args.max_description_token_length is not None: + with accelerator.main_process_first(): + # filter description that is shorter than max_text_length + vectorized_datasets = vectorized_datasets.filter( + lambda x: len(x) < data_args.max_description_token_length, + num_proc=num_workers, + input_columns=["input_ids"], + ) + + if data_args.max_prompt_token_length is not None: + with accelerator.main_process_first(): + # filter description that is shorter than max_text_length + vectorized_datasets = vectorized_datasets.filter( + lambda x: len(x) < data_args.max_prompt_token_length, + num_proc=num_workers, + input_columns=["prompt_input_ids"], + ) + + if data_args.save_to_disk is not None and not dataset_was_precomputed: + if accelerator.is_main_process: + vectorized_datasets.save_to_disk( + data_args.save_to_disk, + num_proc=min(data_args.preprocessing_num_workers, len(vectorized_datasets["eval"]) - 1), + ) + logger.info(f"Dataset saved at {data_args.save_to_disk}") + + audio_max_length = None + if training_args.torch_compile: + audio_max_length = max(vectorized_datasets["train"]["target_length"]) + with accelerator.main_process_first(): + max_sample = vectorized_datasets["train"].filter( + lambda x: x == audio_max_length, + num_proc=num_workers, + input_columns=["target_length"], + ) + audio_max_length = torch.tensor(max_sample[0]["labels"]).shape[1] + + # for large datasets it is advised to run the preprocessing on a + # single machine first with ``args.preprocessing_only`` since there will mostly likely + # be a timeout when running the script in distributed mode. + # In a second step ``args.preprocessing_only`` can then be set to `False` to load the + # cached dataset + if data_args.preprocessing_only and data_args.save_to_disk is None: + raise ValueError( + "`preprocessing_only=True` but `save_to_disk` is not set. The latter should indicates where to save the dataset locally." + ) + elif data_args.preprocessing_only: + logger.info(f"Data preprocessing finished. Files save at {data_args.save_to_disk}") + return + + # 6. Next, we can prepare the training. + + # Let's use word CLAP similary and WER metrics as our evaluation metrics, + + # Define evaluation metrics during training, *i.e.* CLAP similarity + clap = AutoModel.from_pretrained(model_args.clap_model_name_or_path) + clap_processor = AutoProcessor.from_pretrained(model_args.clap_model_name_or_path) + metric = evaluate.load("wer") + + def clap_similarity(texts, audios, device): + clap_inputs = clap_processor(text=texts, audios=audios, padding=True, return_tensors="pt").to(device) + clap.to(device) + with torch.no_grad(): + text_features = clap.get_text_features( + clap_inputs["input_ids"], attention_mask=clap_inputs.get("attention_mask", None) + ) + audio_features = clap.get_audio_features(clap_inputs["input_features"]) + + cosine_sim = torch.nn.functional.cosine_similarity(audio_features, text_features, dim=1, eps=1e-8) + + clap.to("cpu") + clap_inputs.to("cpu") + return cosine_sim.mean().to("cpu") + + def wer(prompts, audios, device): + asr_pipeline = pipeline(model=model_args.asr_model_name_or_path, device=device) + transcriptions = asr_pipeline( + [{"raw": audio, "sampling_rate": sampling_rate} for audio in audios], + batch_size=int(training_args.per_device_eval_batch_size), + ) + + word_error = 100 * metric.compute( + predictions=[t["text"].lower() for t in transcriptions], references=[t.lower() for t in prompts] + ) + + return word_error, [t["text"] for t in transcriptions] + + eval_methods = {"clap": clap_similarity, "wer": wer} + + def compute_metrics(audios, descriptions, prompts, device="cpu"): + input_ids = descriptions + texts = description_tokenizer.batch_decode(input_ids, skip_special_tokens=True) + prompts = prompt_tokenizer.batch_decode(prompts, skip_special_tokens=True) + audios = [a.cpu().numpy() for a in audios] + results = {"clap": eval_methods["clap"](texts, audios, device)} + word_error, transcriptions = eval_methods["wer"](prompts, audios, device) + results["wer"] = word_error + + return results, texts, prompts, audios, transcriptions + + # Define Training Schedule + # Store some constants + per_device_train_batch_size = int(training_args.per_device_train_batch_size) + train_batch_size = per_device_train_batch_size * accelerator.num_processes + gradient_accumulation_steps = int(training_args.gradient_accumulation_steps) + per_device_eval_batch_size = int(training_args.per_device_eval_batch_size) + + if training_args.max_steps < 0: + num_epochs = int(training_args.num_train_epochs) + steps_per_epoch = len(vectorized_datasets["train"]) // (train_batch_size * gradient_accumulation_steps) + total_train_steps = steps_per_epoch * num_epochs + elif training_args.max_steps > 0: + logger.info("max_steps is given, it will override any value given in num_train_epochs") + total_train_steps = int(training_args.max_steps) + # Setting a very large number of epochs so we go as many times as necessary over the iterator. + num_epochs = sys.maxsize + steps_per_epoch = total_train_steps + + if training_args.evaluation_strategy == "epoch": + eval_steps = steps_per_epoch + elif training_args.eval_steps is None: + logger.info(f"eval_steps is not set, evaluating at the end of each epoch") + eval_steps = steps_per_epoch + else: + eval_steps = training_args.eval_steps + + if training_args.save_strategy == "epoch": + save_steps = steps_per_epoch + elif training_args.save_strategy == "steps": + save_steps = training_args.save_steps + else: + save_steps = sys.maxsize + + # T5 doesn't support fp16 + autocast_kwargs = AutocastKwargs(enabled=(mixed_precision != "fp16")) + + # Define optimizer, LR scheduler, collator + optimizer = torch.optim.AdamW( + params=model.parameters(), + lr=training_args.learning_rate, + betas=(training_args.adam_beta1, training_args.adam_beta2), + eps=training_args.adam_epsilon, + weight_decay=training_args.weight_decay, + ) + + # LR scheduler gets stepped by `num_processes` each time -> account for this in warmup / total steps + lr_scheduler = get_scheduler( + name=training_args.lr_scheduler_type, + optimizer=optimizer, + num_warmup_steps=training_args.get_warmup_steps(total_train_steps) * accelerator.num_processes, + num_training_steps=total_train_steps * accelerator.num_processes, + ) + + # Instantiate custom data collator + data_collator = DataCollatorParlerTTSWithPadding( + prompt_tokenizer=prompt_tokenizer, + description_tokenizer=description_tokenizer, + pad_to_multiple_of=data_args.pad_to_multiple_of, + padding=padding, + prompt_max_length=data_args.max_prompt_token_length, + description_max_length=data_args.max_description_token_length, + audio_max_length=audio_max_length, + ) + + # Prepare everything with accelerate + model, optimizer, lr_scheduler = accelerator.prepare(model, optimizer, lr_scheduler) + + logger.info("***** Running training *****") + logger.info(f" Num examples = {total_train_steps * train_batch_size * gradient_accumulation_steps}") + logger.info(" Instantaneous batch size per device =" f" {per_device_train_batch_size}") + logger.info(" Gradient accumulation steps =" f" {gradient_accumulation_steps}") + logger.info( + f" Total train batch size (w. parallel & distributed) = {train_batch_size * gradient_accumulation_steps}" + ) + logger.info(f" Total optimization steps = {total_train_steps}") + + # ======================== Training ================================ + train_time = 0 + train_start = time.time() + steps_trained_progress_bar = tqdm( + range(total_train_steps), desc="Train steps ... ", position=0, disable=not accelerator.is_local_main_process + ) + continue_training = True + epochs_trained = 0 + cur_step = 0 + + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + elif last_checkpoint is not None: + checkpoint = last_checkpoint + + if accelerator.is_main_process: + if training_args.push_to_hub: + # Retrieve of infer repo_name + repo_name = training_args.hub_model_id + if repo_name is None: + repo_name = Path(training_args.output_dir).absolute().name + # Create repo and retrieve repo_id + repo_id = create_repo(repo_name, exist_ok=True, token=training_args.hub_token).repo_id + # Clone repo locally + repo = Repository(training_args.output_dir, clone_from=repo_id, token=training_args.hub_token) + + with open(os.path.join(training_args.output_dir, ".gitignore"), "w+") as gitignore: + if "wandb" not in gitignore: + gitignore.write("wandb\n") + elif training_args.output_dir is not None: + os.makedirs(training_args.output_dir, exist_ok=True) + accelerator.wait_for_everyone() + + # Now save everything to be able to create a single processor later + # make sure all processes wait until data is saved + with accelerator.main_process_first(): + # only the main process saves them + if accelerator.is_main_process: + # save feature extractor, tokenizer and config + if ( + model_args.prompt_tokenizer_name is None + and model_args.description_tokenizer_name + or (model_args.prompt_tokenizer_name == model_args.description_tokenizer_name) + ): + prompt_tokenizer.save_pretrained(training_args.output_dir) + else: + logger.warning( + "Prompt tokenizer ('{model_args.prompt_tokenizer_name}') and description tokenizer ('{model_args.description_tokenizer_name}') are not the same. Saving only the prompt tokenizer." + ) + prompt_tokenizer.save_pretrained(training_args.output_dir) + + feature_extractor.save_pretrained(training_args.output_dir) + config.save_pretrained(training_args.output_dir) + + if checkpoint is not None: + accelerator.load_state(checkpoint) + # Find num steps and epoch from saved state string pattern + pattern = r"checkpoint-(\d+)-epoch-(\d+)" + match = re.search(pattern, checkpoint) + cur_step = int(match.group(1)) + epochs_trained = int(match.group(2)) + + logger.info(" Continuing training from checkpoint, will skip to saved global_step") + logger.info(f" Continuing training from epoch {epochs_trained}") + logger.info(f" Continuing training from global step {cur_step}") + + steps_trained_progress_bar.update(cur_step) + + for epoch in range(0, epochs_trained): + vectorized_datasets["train"] = vectorized_datasets["train"].shuffle(training_args.seed) + + if training_args.max_steps < 0: + # we know exactly the number of steps per epoch, so can skip through the required number of batches + resume_step = (cur_step - epochs_trained * steps_per_epoch) * gradient_accumulation_steps + else: + # Currently we don't know how many steps we've taken in the current epoch + # So we just shuffle the dataset one extra time and start from a fresh epoch + # This is "good enough" for our purposes but not fully correct + resume_step = None + vectorized_datasets["train"] = vectorized_datasets["train"].shuffle(training_args.seed) + else: + resume_step = None + + gen_kwargs = { + "do_sample": model_args.do_sample, + "temperature": model_args.temperature, + "max_length": model_args.max_length, + } + + # Define gradient update step fn + def train_step( + batch, + accelerator, + autocast_kwargs, + ): + model.train() + + if mixed_precision == "fp16": + # fp16 doesn't work with T5-like models + with accelerator.autocast(autocast_handler=autocast_kwargs): + if training_args.parallel_mode.value != "distributed": + encoder_outputs = model.text_encoder( + input_ids=batch.get("input_ids"), attention_mask=batch.get("attention_mask", None) + ) + else: + encoder_outputs = model.module.text_encoder( + input_ids=batch.get("input_ids"), attention_mask=batch.get("attention_mask", None) + ) + batch["encoder_outputs"] = encoder_outputs + + outputs = model(**batch) + # CE (data) loss + ce_loss = outputs.loss + + metrics = {"loss": ce_loss} + return ce_loss, metrics + + # Define eval fn + def eval_step( + batch, + accelerator, + autocast_kwargs, + ): + eval_model = model if not training_args.torch_compile else model._orig_mod + eval_model.eval() + + if mixed_precision == "fp16": + # fp16 doesn't work with T5-like models + with accelerator.autocast(autocast_handler=autocast_kwargs): + with torch.no_grad(): + if training_args.parallel_mode.value != "distributed" or training_args.torch_compile: + encoder_outputs = eval_model.text_encoder( + input_ids=batch.get("input_ids"), attention_mask=batch.get("attention_mask", None) + ) + else: + encoder_outputs = eval_model.module.text_encoder( + input_ids=batch.get("input_ids"), attention_mask=batch.get("attention_mask", None) + ) + batch["encoder_outputs"] = encoder_outputs + + with torch.no_grad(): + outputs = eval_model(**batch) + # CE (data) loss + ce_loss = outputs.loss + metrics = {"loss": ce_loss} + return metrics + + def generate_step(batch): + batch.pop("decoder_attention_mask", None) + eval_model = accelerator.unwrap_model(model, keep_fp32_wrapper=mixed_precision != "fp16").eval() + if training_args.torch_compile: + eval_model = model._orig_mod + + output_audios = eval_model.generate(**batch, **gen_kwargs) + output_audios = accelerator.pad_across_processes(output_audios, dim=1, pad_index=0) + return output_audios + + for epoch in range(epochs_trained, num_epochs): + vectorized_datasets["train"] = vectorized_datasets["train"].shuffle(training_args.seed) + sampler = None + if training_args.group_by_length: + sampler = LengthGroupedSampler(train_batch_size, lengths=vectorized_datasets["train"]["target_length"]) + train_dataloader = DataLoader( + vectorized_datasets["train"], + collate_fn=data_collator, + batch_size=per_device_train_batch_size, + sampler=sampler, + num_workers=training_args.dataloader_num_workers, + pin_memory=training_args.dataloader_pin_memory, + ) + train_dataloader = accelerator.prepare(train_dataloader) + if hasattr(train_dataloader, "dataset") and isinstance(train_dataloader.dataset, IterableDataset): + train_dataloader.dataset.set_epoch(epoch) + + if resume_step is not None: + # Skip the first N batches in the dataloader when resuming from a checkpoint + train_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step) + resume_step = None + + for batch in train_dataloader: + with accelerator.accumulate(model): + loss, train_metric = train_step(batch, accelerator, autocast_kwargs) + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(model.parameters(), training_args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Check if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + steps_trained_progress_bar.update(1) + cur_step += 1 + + if cur_step % training_args.logging_steps == 0: + steps_trained_progress_bar.write( + f"Step... ({cur_step} / {total_train_steps} | Loss:" + f" {train_metric['loss']}, Learning Rate:" + f" {lr_scheduler.get_last_lr()[0]})" + ) + log_metric( + accelerator, + metrics=train_metric, + learning_rate=lr_scheduler.get_last_lr()[0], + train_time=train_time + time.time() - train_start, + step=cur_step, + epoch=epoch + (cur_step - epoch * steps_per_epoch) / steps_per_epoch, + prefix="train", + ) + + # save checkpoint and weights after each save_steps and at the end of training + if (cur_step % save_steps == 0) or cur_step == total_train_steps: + intermediate_dir = os.path.join(training_args.output_dir, f"checkpoint-{cur_step}-epoch-{epoch}") + # safe_serialization=False to avoid shared tensors saving issue (TODO(YL): it's a temporary fix) + # https://github.com/huggingface/transformers/issues/27293#issuecomment-1872560074 + accelerator.save_state(output_dir=intermediate_dir, safe_serialization=False) + accelerator.wait_for_everyone() + if accelerator.is_main_process: + rotate_checkpoints(training_args.save_total_limit, output_dir=training_args.output_dir) + + if cur_step == total_train_steps: + # un-wrap student model for save + unwrapped_model = accelerator.unwrap_model(model) + unwrapped_model.save_pretrained(training_args.output_dir) + + if training_args.push_to_hub: + repo.push_to_hub( + commit_message=f"Saving train state of step {cur_step}", + blocking=False, + ) + + if training_args.do_eval and (cur_step % eval_steps == 0 or cur_step == total_train_steps): + train_time += time.time() - train_start + # ======================== Evaluating ============================== + eval_metrics = [] + eval_preds = [] + eval_descriptions = [] + eval_prompts = [] + eval_start = time.time() + + # release training input batch + batch = release_memory(batch) + + validation_dataloader = DataLoader( + vectorized_datasets["eval"], + collate_fn=data_collator, + batch_size=per_device_eval_batch_size, + drop_last=False, + num_workers=training_args.dataloader_pin_memory, + pin_memory=training_args.dataloader_pin_memory, + ) + validation_dataloader = accelerator.prepare(validation_dataloader) + + for batch in tqdm( + validation_dataloader, + desc="Evaluating - Inference ...", + position=2, + disable=not accelerator.is_local_main_process, + ): + # Model forward + eval_metric = eval_step(batch, accelerator, autocast_kwargs) + eval_metric = accelerator.gather_for_metrics(eval_metric) + eval_metrics.append(eval_metric) + + if training_args.predict_with_generate: + validation_dataloader = DataLoader( + vectorized_datasets["eval"], + collate_fn=data_collator, + batch_size=per_device_eval_batch_size, + drop_last=False, + num_workers=training_args.dataloader_pin_memory, + pin_memory=training_args.dataloader_pin_memory, + ) + validation_dataloader = accelerator.prepare(validation_dataloader) + # generation + for batch in tqdm( + validation_dataloader, + desc="Evaluating - Generation ...", + position=2, + disable=not accelerator.is_local_main_process, + ): + generated_audios = generate_step(batch) + # Gather all predictions and targets + generated_audios, input_ids, prompts = accelerator.pad_across_processes( + (generated_audios, batch["input_ids"], batch["prompt_input_ids"]), dim=1, pad_index=0 + ) + generated_audios, input_ids, prompts = accelerator.gather_for_metrics( + (generated_audios, input_ids, prompts) + ) + eval_preds.extend(generated_audios.to("cpu")) + eval_descriptions.extend(input_ids.to("cpu")) + eval_prompts.extend(prompts.to("cpu")) + + eval_time = time.time() - eval_start + # normalize eval metrics + eval_metrics = { + key: torch.mean(torch.cat([d[key].unsqueeze(0) for d in eval_metrics])) + for key in eval_metrics[0] + } + + # compute metrics + metrics_desc = "" + if training_args.predict_with_generate: + metric_values, pred_descriptions, pred_prompts, audios, transcriptions = compute_metrics( + eval_preds, eval_descriptions, eval_prompts, accelerator.device + ) + eval_metrics.update(metric_values) + metrics_desc = " ".join([f"Eval {key}: {value} |" for key, value in metric_values.items()]) + if "wandb" in training_args.report_to: + log_pred( + accelerator, + pred_descriptions, + pred_prompts, + transcriptions, + audios, + sampling_rate=sampling_rate, + step=cur_step, + prefix="eval", + ) + + # Print metrics and update progress bar + steps_trained_progress_bar.write( + f"Eval results for step ({cur_step} / {total_train_steps} | Eval Loss: {eval_metrics['loss']} |" + f" {metrics_desc})" + ) + + log_metric( + accelerator, + metrics=eval_metrics, + train_time=eval_time, + step=cur_step, + epoch=epoch + (cur_step - epoch * steps_per_epoch) / steps_per_epoch, + prefix="eval", + ) + + # release eval batch and relax metrics + eval_metrics = [] + eval_preds = [] + eval_descriptions = [] + eval_prompts = [] + batch = release_memory(batch) + + # flush the train metrics + train_start = time.time() + + # break condition + if cur_step == total_train_steps: + continue_training = False + break + + if not continue_training: + break + + accelerator.end_training() + + +if __name__ == "__main__": + set_start_method("spawn") + main() diff --git a/wandb/debug-cli.sanchit.log b/wandb/debug-cli.sanchit.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..10fc533ac41b82f6c553d78e06ac3eca75321b6b --- /dev/null +++ b/wandb/debug-internal.log @@ -0,0 +1,11809 @@ +2024-05-13 20:52:49,022 INFO StreamThr :1257956 [internal.py:wandb_internal():85] W&B internal server running at pid: 1257956, started at: 2024-05-13 20:52:49.021573 +2024-05-13 20:52:49,024 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status +2024-05-13 20:52:49,026 INFO WriterThread:1257956 [datastore.py:open_for_write():87] open: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/run-qaoje1x9.wandb +2024-05-13 20:52:49,027 DEBUG SenderThread:1257956 [sender.py:send():378] send: header +2024-05-13 20:52:49,027 DEBUG SenderThread:1257956 [sender.py:send():378] send: run +2024-05-13 20:52:49,408 INFO SenderThread:1257956 [dir_watcher.py:__init__():211] watching files in: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files +2024-05-13 20:52:49,408 INFO SenderThread:1257956 [sender.py:_start_run_threads():1123] run started: qaoje1x9 with start time 1715626369.020078 +2024-05-13 20:52:49,414 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: check_version +2024-05-13 20:52:49,414 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: check_version +2024-05-13 20:52:49,480 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: run_start +2024-05-13 20:52:49,502 DEBUG HandlerThread:1257956 [system_info.py:__init__():26] System info init +2024-05-13 20:52:49,502 DEBUG HandlerThread:1257956 [system_info.py:__init__():41] System info init done +2024-05-13 20:52:49,502 INFO HandlerThread:1257956 [system_monitor.py:start():194] Starting system monitor +2024-05-13 20:52:49,502 INFO SystemMonitor:1257956 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-13 20:52:49,502 INFO HandlerThread:1257956 [system_monitor.py:probe():214] Collecting system info +2024-05-13 20:52:49,503 INFO SystemMonitor:1257956 [interfaces.py:start():188] Started cpu monitoring +2024-05-13 20:52:49,503 INFO SystemMonitor:1257956 [interfaces.py:start():188] Started disk monitoring +2024-05-13 20:52:49,504 INFO SystemMonitor:1257956 [interfaces.py:start():188] Started gpu monitoring +2024-05-13 20:52:49,505 INFO SystemMonitor:1257956 [interfaces.py:start():188] Started memory monitoring +2024-05-13 20:52:49,506 INFO SystemMonitor:1257956 [interfaces.py:start():188] Started network monitoring +2024-05-13 20:52:49,548 DEBUG HandlerThread:1257956 [system_info.py:probe():150] Probing system +2024-05-13 20:52:49,550 DEBUG HandlerThread:1257956 [system_info.py:_probe_git():135] Probing git +2024-05-13 20:52:49,555 DEBUG HandlerThread:1257956 [system_info.py:_probe_git():143] Probing git done +2024-05-13 20:52:49,555 DEBUG HandlerThread:1257956 [system_info.py:probe():198] Probing system done +2024-05-13 20:52:49,555 DEBUG HandlerThread:1257956 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.31', 'python': '3.11.9', 'heartbeatAt': '2024-05-13T18:52:49.548604', 'startedAt': '2024-05-13T18:52:49.014517', 'docker': None, 'cuda': None, 'args': ('finetuning_concatenated_config.json',), 'state': 'running', 'program': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py', 'codePathLocal': 'run_parler_tts_training.py', 'codePath': 'run_parler_tts_training.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/parler-tts-mini-v0.1-expresso-concatenated-combined', 'commit': '50ba4323d7b8bb052629aa1b88283b9df081a821'}, 'email': 'sanchit@huggingface.co', 'root': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/miniconda3/envs/venv/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2360.7500156249994, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 2426.568, 'min': 1500.0, 'max': 2250.0}, {'current': 2649.263, 'min': 1500.0, 'max': 2250.0}, {'current': 3020.85, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.979, 'min': 1500.0, 'max': 2250.0}, {'current': 3294.864, 'min': 1500.0, 'max': 2250.0}, {'current': 2097.135, 'min': 1500.0, 'max': 2250.0}, {'current': 1662.773, 'min': 1500.0, 'max': 2250.0}, {'current': 1664.784, 'min': 1500.0, 'max': 2250.0}, {'current': 3332.909, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.756, 'min': 1500.0, 'max': 2250.0}, {'current': 1583.69, 'min': 1500.0, 'max': 2250.0}, {'current': 3225.763, 'min': 1500.0, 'max': 2250.0}, {'current': 1713.916, 'min': 1500.0, 'max': 2250.0}, {'current': 2383.108, 'min': 1500.0, 'max': 2250.0}, {'current': 1713.839, 'min': 1500.0, 'max': 2250.0}, {'current': 1712.972, 'min': 1500.0, 'max': 2250.0}, {'current': 2360.612, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.124, 'min': 1500.0, 'max': 2250.0}, {'current': 3286.553, 'min': 1500.0, 'max': 2250.0}, {'current': 3279.781, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.151, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.308, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.785, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.778, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.19, 'min': 1500.0, 'max': 2250.0}, {'current': 3194.481, 'min': 1500.0, 'max': 2250.0}, {'current': 3332.944, 'min': 1500.0, 'max': 2250.0}, {'current': 3210.408, 'min': 1500.0, 'max': 2250.0}, {'current': 2661.36, 'min': 1500.0, 'max': 2250.0}, {'current': 1716.914, 'min': 1500.0, 'max': 2250.0}, {'current': 1712.659, 'min': 1500.0, 'max': 2250.0}, {'current': 1717.873, 'min': 1500.0, 'max': 2250.0}, {'current': 1797.636, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.013, 'min': 1500.0, 'max': 2250.0}, {'current': 2777.317, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.753, 'min': 1500.0, 'max': 2250.0}, {'current': 3297.674, 'min': 1500.0, 'max': 2250.0}, {'current': 2290.065, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.248, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.016, 'min': 1500.0, 'max': 2250.0}, {'current': 3299.101, 'min': 1500.0, 'max': 2250.0}, {'current': 1647.783, 'min': 1500.0, 'max': 2250.0}, {'current': 1660.513, 'min': 1500.0, 'max': 2250.0}, {'current': 1659.949, 'min': 1500.0, 'max': 2250.0}, {'current': 3318.509, 'min': 1500.0, 'max': 2250.0}, {'current': 2655.27, 'min': 1500.0, 'max': 2250.0}, {'current': 3320.021, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.518, 'min': 1500.0, 'max': 2250.0}, {'current': 3341.592, 'min': 1500.0, 'max': 2250.0}, {'current': 1662.725, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.592, 'min': 1500.0, 'max': 2250.0}, {'current': 1662.043, 'min': 1500.0, 'max': 2250.0}, {'current': 1714.097, 'min': 1500.0, 'max': 2250.0}, {'current': 3186.226, 'min': 1500.0, 'max': 2250.0}, {'current': 1712.114, 'min': 1500.0, 'max': 2250.0}, {'current': 1718.52, 'min': 1500.0, 'max': 2250.0}, {'current': 1701.556, 'min': 1500.0, 'max': 2250.0}, {'current': 3294.711, 'min': 1500.0, 'max': 2250.0}, {'current': 3324.584, 'min': 1500.0, 'max': 2250.0}, {'current': 1659.515, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.143, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.233, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.995, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.883, 'min': 1500.0, 'max': 2250.0}, {'current': 1790.024, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.313, 'min': 1500.0, 'max': 2250.0}, {'current': 2988.012, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.457, 'min': 1500.0, 'max': 2250.0}, {'current': 3328.716, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.634, 'min': 1500.0, 'max': 2250.0}, {'current': 1662.011, 'min': 1500.0, 'max': 2250.0}, {'current': 2209.287, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.369, 'min': 1500.0, 'max': 2250.0}, {'current': 1658.97, 'min': 1500.0, 'max': 2250.0}, {'current': 1662.145, 'min': 1500.0, 'max': 2250.0}, {'current': 2424.213, 'min': 1500.0, 'max': 2250.0}, {'current': 2703.578, 'min': 1500.0, 'max': 2250.0}, {'current': 2305.348, 'min': 1500.0, 'max': 2250.0}, {'current': 1716.951, 'min': 1500.0, 'max': 2250.0}, {'current': 1718.102, 'min': 1500.0, 'max': 2250.0}, {'current': 3140.126, 'min': 1500.0, 'max': 2250.0}, {'current': 3317.716, 'min': 1500.0, 'max': 2250.0}, {'current': 3313.497, 'min': 1500.0, 'max': 2250.0}, {'current': 3316.184, 'min': 1500.0, 'max': 2250.0}, {'current': 1809.855, 'min': 1500.0, 'max': 2250.0}, {'current': 3078.611, 'min': 1500.0, 'max': 2250.0}, {'current': 3080.162, 'min': 1500.0, 'max': 2250.0}, {'current': 2037.58, 'min': 1500.0, 'max': 2250.0}, {'current': 2198.863, 'min': 1500.0, 'max': 2250.0}, {'current': 3263.675, 'min': 1500.0, 'max': 2250.0}, {'current': 3293.238, 'min': 1500.0, 'max': 2250.0}, {'current': 2189.229, 'min': 1500.0, 'max': 2250.0}, {'current': 2038.013, 'min': 1500.0, 'max': 2250.0}, {'current': 2217.719, 'min': 1500.0, 'max': 2250.0}, {'current': 3135.886, 'min': 1500.0, 'max': 2250.0}, {'current': 2193.004, 'min': 1500.0, 'max': 2250.0}, {'current': 1805.953, 'min': 1500.0, 'max': 2250.0}, {'current': 1832.919, 'min': 1500.0, 'max': 2250.0}, {'current': 1831.97, 'min': 1500.0, 'max': 2250.0}, {'current': 1831.656, 'min': 1500.0, 'max': 2250.0}, {'current': 3312.099, 'min': 1500.0, 'max': 2250.0}, {'current': 2035.327, 'min': 1500.0, 'max': 2250.0}, {'current': 1957.443, 'min': 1500.0, 'max': 2250.0}, {'current': 2216.063, 'min': 1500.0, 'max': 2250.0}, {'current': 3022.623, 'min': 1500.0, 'max': 2250.0}, {'current': 1684.605, 'min': 1500.0, 'max': 2250.0}, {'current': 1684.549, 'min': 1500.0, 'max': 2250.0}, {'current': 1702.372, 'min': 1500.0, 'max': 2250.0}, {'current': 3286.386, 'min': 1500.0, 'max': 2250.0}, {'current': 3288.738, 'min': 1500.0, 'max': 2250.0}, {'current': 3278.751, 'min': 1500.0, 'max': 2250.0}, {'current': 3290.555, 'min': 1500.0, 'max': 2250.0}, {'current': 3292.535, 'min': 1500.0, 'max': 2250.0}, {'current': 2153.843, 'min': 1500.0, 'max': 2250.0}, {'current': 3293.152, 'min': 1500.0, 'max': 2250.0}, {'current': 1644.116, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.185, 'min': 1500.0, 'max': 2250.0}, {'current': 3070.901, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.504, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.465, 'min': 1500.0, 'max': 2250.0}, {'current': 1719.19, 'min': 1500.0, 'max': 2250.0}, {'current': 3284.274, 'min': 1500.0, 'max': 2250.0}, {'current': 3283.49, 'min': 1500.0, 'max': 2250.0}, {'current': 1645.649, 'min': 1500.0, 'max': 2250.0}, {'current': 1790.709, 'min': 1500.0, 'max': 2250.0}, {'current': 2223.188, 'min': 1500.0, 'max': 2250.0}, {'current': 1797.667, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.254, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1663.5053825378418}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} +2024-05-13 20:52:49,556 INFO HandlerThread:1257956 [system_monitor.py:probe():224] Finished collecting system info +2024-05-13 20:52:49,556 INFO HandlerThread:1257956 [system_monitor.py:probe():227] Publishing system info +2024-05-13 20:52:49,556 DEBUG HandlerThread:1257956 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-13 20:52:49,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:49,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:50,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/conda-environment.yaml +2024-05-13 20:52:51,976 DEBUG HandlerThread:1257956 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-13 20:52:51,977 INFO HandlerThread:1257956 [system_monitor.py:probe():229] Finished publishing system info +2024-05-13 20:52:51,985 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 20:52:51,985 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-metadata.json with policy now +2024-05-13 20:52:52,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: python_packages +2024-05-13 20:52:52,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:52:52,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: python_packages +2024-05-13 20:52:52,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:52:52,322 DEBUG SenderThread:1257956 [sender.py:send():378] send: telemetry +2024-05-13 20:52:52,323 DEBUG SenderThread:1257956 [sender.py:send():378] send: config +2024-05-13 20:52:52,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/conda-environment.yaml +2024-05-13 20:52:52,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/requirements.txt +2024-05-13 20:52:52,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:52:52,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-metadata.json +2024-05-13 20:52:52,549 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ht5mod1s-wandb-metadata.json +2024-05-13 20:52:52,583 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:52,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:54,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:52:54,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:52:54,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:54,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:56,410 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:52:56,662 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:56,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:58,410 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:52:59,391 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:59,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:00,001 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:00,411 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:01,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:01,488 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:02,411 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:04,517 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:04,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:05,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:06,413 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:06,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:06,606 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:07,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:53:07,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:53:08,413 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:09,439 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:09,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:10,414 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:11,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:11,492 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:11,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:12,414 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:14,329 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:14,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:16,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:16,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:16,416 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:17,037 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:18,417 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:19,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:19,437 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:20,417 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:21,488 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:21,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:22,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:53:22,103 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:53:22,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:22,418 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:24,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:24,418 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:24,418 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/config.yaml +2024-05-13 20:53:24,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:26,419 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:28,231 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:53:28,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:28,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:28,419 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:29,250 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:30,420 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:31,218 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:31,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:32,420 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:33,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:33,316 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:35,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:36,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:36,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:36,422 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:37,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:53:37,103 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:53:38,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:38,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:38,422 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:40,277 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:41,397 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:41,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:42,423 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:43,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:43,460 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:44,424 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:46,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:46,761 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:46,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:48,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:48,925 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:49,506 DEBUG SystemMonitor:1257956 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-13 20:53:49,510 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:53:50,425 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:50,968 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:50,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:51,632 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:52,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:53:52,103 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:53:52,426 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:53,050 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:53,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:56,208 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:56,234 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:56,427 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:56,713 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:58,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:58,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:58,428 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:00,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:00,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:00,428 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:02,417 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:02,457 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:02,685 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:06,430 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:06,506 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:54:06,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:07,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:54:07,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:54:07,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:08,298 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:09,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:09,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:10,431 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:11,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:12,004 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:12,432 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:13,583 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:14,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:14,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:14,433 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:16,085 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:16,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:16,433 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:18,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:18,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:18,434 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:18,599 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:19,512 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:54:20,171 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:20,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:20,435 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:22,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:54:22,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:54:22,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:22,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:22,435 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:23,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:24,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:24,298 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:24,436 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:26,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:26,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:26,436 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:28,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:28,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:28,437 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:28,748 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:30,437 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:30,437 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:30,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:32,438 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:32,493 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:32,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:33,845 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:34,439 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:34,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:34,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:36,439 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:36,864 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:36,884 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:37,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:54:37,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:54:38,440 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:38,906 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:40,368 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:54:40,440 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:41,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:41,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:42,441 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:43,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:43,506 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:44,057 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:44,442 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:45,533 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:45,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:46,442 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:47,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:47,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:48,443 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:49,393 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:49,514 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:54:49,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:49,636 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:50,443 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:51,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:51,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:52,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:54:52,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:54:52,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:53,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:53,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:54,445 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:54,447 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:56,107 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:56,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:56,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:58,373 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:58,408 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:58,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:59,538 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:00,447 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:00,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:00,608 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:02,449 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:02,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:02,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:04,451 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:04,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:04,959 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:05,004 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:06,454 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:07,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:55:07,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:55:07,138 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:07,162 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:08,455 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:09,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:09,337 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:09,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:10,457 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:11,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:11,487 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:12,458 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:14,460 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:14,935 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:15,713 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:55:15,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:15,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:16,463 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:17,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:17,863 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:18,463 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:19,518 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:55:19,942 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:19,965 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:20,044 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:20,465 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:22,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:55:22,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:55:22,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:22,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:22,467 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:24,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:24,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:24,469 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:25,093 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:26,305 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:26,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:26,472 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:28,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:28,429 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:28,473 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:30,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:30,475 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:30,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:30,643 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:32,476 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:32,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:32,850 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:34,479 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:35,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:35,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:35,355 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:36,481 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:37,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:55:37,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:55:37,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:37,224 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:38,482 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:39,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:39,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:40,443 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:40,484 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:41,485 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:41,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:42,486 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:43,542 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:43,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:44,487 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:45,544 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:45,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:45,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:46,490 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:48,492 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:49,520 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:55:49,986 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:55:50,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:50,050 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:50,494 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:50,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:52,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:55:52,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:55:52,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:52,242 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:52,495 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:54,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:54,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:54,497 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:55,845 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:56,499 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:56,574 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:56,608 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:58,500 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:58,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:58,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:00,501 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:00,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:00,720 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:00,925 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:02,502 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:02,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:02,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:04,503 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:04,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:04,809 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:06,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:06,506 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:06,832 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:06,845 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:07,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:56:07,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:56:08,507 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:08,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:08,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:10,509 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:10,917 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:10,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:12,056 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:12,511 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:14,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:14,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:14,512 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:16,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:16,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:16,515 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:17,062 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:18,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:18,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:18,516 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:19,522 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:56:20,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:20,172 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:20,517 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:22,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:56:22,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:56:22,337 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:22,518 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:24,520 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:25,183 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:56:25,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:25,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:26,523 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:27,278 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:27,291 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:27,543 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:28,524 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:30,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:30,437 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:30,525 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:32,456 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:32,466 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:32,527 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:32,647 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:34,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:34,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:34,529 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:36,532 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:36,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:36,563 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:37,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:56:37,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:56:37,842 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:38,533 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:39,488 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:39,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:40,534 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:41,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:41,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:42,536 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:42,892 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:44,538 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:44,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:44,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:46,540 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:46,659 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:46,671 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:47,973 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:48,542 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:49,524 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:56:49,958 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:49,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:50,543 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:52,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:52,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:52,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:56:52,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:56:52,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:53,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:54,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:54,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:54,546 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:56,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:56,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:56,550 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:58,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:58,178 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:58,224 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:58,551 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:00,553 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:01,977 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:57:02,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:02,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:02,554 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:03,276 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:04,556 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:05,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:05,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:06,559 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:07,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:07,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:07,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:07,121 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:08,424 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:08,560 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:09,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:09,152 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:10,562 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:11,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:11,198 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:12,564 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:13,498 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:14,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:14,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:14,565 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:16,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:16,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:16,568 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:18,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:18,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:18,570 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:18,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:19,526 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:57:20,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:20,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:20,571 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:22,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:22,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:22,573 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:23,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:23,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:23,818 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:24,574 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:25,702 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:25,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:26,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:27,740 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:27,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:28,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:28,870 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:29,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:29,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:30,580 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:32,582 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:33,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:33,128 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:33,972 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:34,584 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:35,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:35,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:36,586 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:37,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:37,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:38,587 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:39,157 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:39,194 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:57:39,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:39,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:40,588 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:41,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:41,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:42,590 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:43,298 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:43,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:44,246 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:44,591 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:46,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:46,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:46,594 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:48,576 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:48,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:48,595 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:49,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:49,530 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:57:50,596 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:50,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:50,638 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:52,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:52,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:52,598 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:53,896 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:53,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:54,420 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:54,599 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:55,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:55,946 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:56,601 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:57,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:57,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:58,602 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:59,541 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:00,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:00,019 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:00,603 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:02,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:02,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:02,605 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:04,606 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:04,661 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:05,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:05,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:06,608 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:07,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:07,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:07,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:07,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:08,610 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:09,488 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:09,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:09,733 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:10,612 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:11,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:11,547 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:12,614 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:14,615 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:14,886 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:15,734 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:58:15,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:15,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:16,618 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:17,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:18,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:18,619 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:19,532 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:58:19,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:20,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:20,149 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:20,621 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:22,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:22,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:22,325 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:22,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:22,622 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:24,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:24,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:24,624 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:25,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:26,626 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:26,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:26,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:28,627 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:28,847 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:28,870 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:30,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:30,629 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:31,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:31,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:32,630 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:33,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:33,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:34,632 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:35,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:35,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:35,440 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:36,634 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:37,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:37,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:37,525 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:37,560 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:38,635 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:39,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:39,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:40,243 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:40,636 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:41,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:42,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:42,638 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:44,185 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:44,211 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:44,639 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:45,258 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:46,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:46,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:46,642 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:48,643 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:49,141 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:58:49,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:49,535 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:58:49,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:50,327 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:50,645 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:52,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:52,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:52,646 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:52,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:52,924 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:54,648 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:55,144 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:55,162 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:55,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:56,650 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:57,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:57,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:58,651 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:59,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:59,528 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:00,451 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:00,652 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:01,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:01,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:02,654 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:03,917 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:03,938 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:04,655 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:05,500 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:06,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:06,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:06,656 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:07,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:07,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:08,228 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:08,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:08,658 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:10,392 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:10,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:10,536 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:10,660 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:12,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:12,624 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:12,662 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:14,663 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:14,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:14,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:15,582 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:16,665 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:17,015 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:17,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:18,667 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:19,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:19,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:19,537 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:59:20,668 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:20,710 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:21,358 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:21,377 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:22,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:22,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:22,670 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:24,671 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:25,199 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:59:25,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:25,729 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:25,743 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:26,673 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:27,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:27,914 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:28,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:30,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:30,119 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:30,676 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:30,822 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:32,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:32,267 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:32,678 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:34,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:34,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:34,679 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:35,830 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:36,681 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:36,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:36,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:37,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:37,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:38,683 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:38,872 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:38,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:40,684 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:40,900 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:41,042 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:41,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:42,686 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:43,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:43,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:44,688 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:45,417 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:45,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:45,994 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:46,691 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:47,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:47,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:48,693 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:49,540 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:59:49,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:49,809 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:50,694 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:51,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:51,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:51,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:52,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:52,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:52,696 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:54,159 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:54,172 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:54,698 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:56,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:56,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:56,400 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:56,700 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:58,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:59,832 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:00:00,701 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:00,703 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:00,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:01,553 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:02,705 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:02,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:02,990 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:04,707 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:05,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:05,049 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:06,663 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:06,709 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:07,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:07,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:07,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:07,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:08,710 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:09,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:09,174 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:10,712 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:11,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:11,322 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:11,798 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:12,714 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:13,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:13,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:14,715 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:15,730 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:15,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:16,719 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:16,981 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:17,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:17,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:18,720 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:19,542 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:00:19,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:19,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:20,722 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:22,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:22,064 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:22,072 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:22,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:22,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:22,724 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:24,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:24,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:24,726 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:26,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:26,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:26,728 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:27,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:28,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:28,684 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:28,729 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:30,731 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:30,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:30,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:32,270 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:32,733 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:34,735 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:35,138 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:00:35,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:35,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:36,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:37,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:37,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:37,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:37,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:37,412 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:38,738 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:39,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:39,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:40,740 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:41,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:41,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:42,497 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:42,742 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:43,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:43,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:44,744 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:46,121 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:46,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:46,747 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:47,574 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:48,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:48,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:48,749 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:49,545 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:00:50,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:50,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:50,751 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:52,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:52,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:52,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:52,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:52,753 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:52,845 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:54,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:54,469 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:54,754 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:56,497 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:56,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:56,756 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:57,888 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:58,552 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:58,563 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:58,758 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:00,759 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:01,722 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:01,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:02,760 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:02,914 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:03,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:03,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:04,762 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:05,932 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:05,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:06,763 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:07,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:07,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:08,016 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:08,765 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:09,930 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:01:09,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:09,979 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:10,766 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:12,014 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:12,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:12,768 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:13,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:14,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:14,077 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:14,770 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:16,773 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:17,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:17,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:18,165 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:18,774 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:19,547 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:01:20,100 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:20,134 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:20,776 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:22,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:22,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:22,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:22,287 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:22,778 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:23,203 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:24,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:24,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:24,780 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:26,781 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:27,570 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:27,598 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:28,290 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:28,782 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:29,677 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:29,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:30,784 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:31,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:31,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:32,786 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:33,383 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:33,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:33,807 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:34,787 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:35,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:35,839 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:36,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:37,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:37,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:37,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:37,949 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:38,487 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:38,790 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:39,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:40,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:40,792 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:42,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:42,045 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:42,793 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:43,590 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:44,795 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:46,019 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:01:46,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:46,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:46,799 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:48,077 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:48,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:48,660 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:48,800 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:49,550 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:01:50,802 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:51,301 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:51,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:52,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:52,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:52,804 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:53,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:53,403 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:54,464 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:54,805 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:55,438 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:55,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:58,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:58,382 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:58,807 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:59,591 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:00,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:00,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:00,809 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:02,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:02,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:02,810 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:04,689 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:04,747 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:04,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:04,812 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:06,814 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:06,990 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:07,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:07,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:07,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:08,815 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:09,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:09,226 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:09,794 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:10,817 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:11,343 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:11,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:12,818 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:13,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:13,528 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:14,820 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:15,529 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:15,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:15,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:16,823 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:17,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:17,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:19,552 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:02:20,554 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:21,590 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:02:21,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:21,653 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:22,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:22,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:22,825 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:24,829 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:24,917 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:24,938 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:26,060 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:26,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:27,000 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:28,831 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:29,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:29,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:30,833 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:31,068 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:31,078 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:31,092 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:32,835 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:33,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:33,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:34,837 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:36,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:36,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:36,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:36,839 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:37,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:37,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:38,840 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:39,374 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:39,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:40,842 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:41,235 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:41,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:41,517 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:42,844 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:43,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:43,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:44,845 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:46,250 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:46,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:46,816 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:46,849 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:48,847 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:48,851 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:48,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:49,555 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:02:50,853 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:50,908 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:50,921 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:51,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:52,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:52,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:52,855 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:52,955 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:52,993 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:54,856 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:55,051 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:55,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:56,458 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:56,858 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:58,861 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:00,608 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:03:00,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:00,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:00,861 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:01,510 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:02,728 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:02,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:02,863 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:04,810 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:04,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:04,865 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:06,533 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:06,866 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:06,877 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:06,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:07,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:07,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:08,868 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:08,949 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:08,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:10,870 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:11,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:11,051 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:11,544 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:12,872 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:13,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:13,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:14,874 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:15,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:15,176 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:16,623 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:16,877 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:18,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:18,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:18,879 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:19,560 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:03:20,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:20,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:20,881 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:21,781 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:22,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:22,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:22,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:22,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:22,883 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:24,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:24,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:24,885 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:26,851 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:26,887 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:27,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:27,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:28,888 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:29,425 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:29,440 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:30,890 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:31,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:31,495 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:31,887 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:32,892 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:34,894 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:35,424 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:03:35,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:35,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:36,896 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:37,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:37,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:37,317 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:38,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:38,797 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:40,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:40,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:40,898 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:42,886 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:42,887 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:42,900 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:42,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:44,925 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:44,942 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:46,903 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:48,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:48,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:48,338 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:49,560 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:03:50,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:50,402 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:50,904 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:52,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:52,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:52,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:52,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:52,906 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:54,331 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:55,566 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:55,593 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:56,908 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:57,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:57,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:58,910 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:59,587 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:59,696 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:59,709 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:01,737 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:01,747 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:03,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:03,782 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:04,820 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:04,912 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:05,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:05,811 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:06,914 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:07,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:07,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:07,832 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:07,842 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:08,915 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:10,299 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:10,916 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:13,207 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:04:13,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:13,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:15,301 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:15,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:15,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:17,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:17,493 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:19,562 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:04:19,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:19,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:20,563 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:21,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:21,729 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:22,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:22,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:23,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:23,834 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:25,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:25,982 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:26,264 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:28,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:28,094 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:28,920 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:30,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:30,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:32,079 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:32,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:32,377 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:34,496 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:34,525 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:36,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:36,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:36,923 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:37,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:37,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:37,286 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:38,802 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:38,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:40,925 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:40,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:40,994 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:42,567 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:43,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:43,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:44,927 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:46,968 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:04:46,969 DEBUG SenderThread:1257956 [sender.py:send():378] send: telemetry +2024-05-13 21:04:47,208 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:04:47,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:47,290 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:47,972 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:48,929 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:48,929 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/config.yaml +2024-05-13 21:04:49,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:49,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:49,565 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:04:50,930 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:51,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:51,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:52,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:52,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:53,263 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:53,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:53,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:55,470 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:55,480 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:57,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:57,516 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:58,265 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:59,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:59,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:01,571 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:01,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:03,266 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:03,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:03,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:05,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:05,753 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:06,935 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:07,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:07,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:07,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:07,794 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:08,273 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:09,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:09,841 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:11,896 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:11,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:13,274 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:13,984 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:14,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:14,937 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:16,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:16,088 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:18,934 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:19,568 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:05:20,127 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:05:20,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:20,212 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:20,939 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:22,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:22,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:22,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:22,274 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:23,982 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:23,983 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:05:23,984 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:05:23,984 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:05:23,984 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:05:24,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:24,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:24,941 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:24,941 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:05:26,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:26,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:28,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:28,476 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:28,943 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:28,990 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:30,520 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:30,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:30,945 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:32,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:32,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:34,262 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:34,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:34,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:36,705 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:36,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:37,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:37,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:38,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:38,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:39,319 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:40,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:40,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:42,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:43,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:44,877 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:44,949 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:45,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:45,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:47,276 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:47,309 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:49,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:49,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:49,569 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:05:50,570 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:52,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:52,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:53,491 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:05:53,560 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:53,580 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:54,952 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:55,666 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:55,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:56,392 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:57,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:57,824 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:59,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:05:59,650 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:05:59,650 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:05:59,650 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:05:59,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:59,954 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:05:59,958 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:00,954 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:01,651 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:02,102 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:02,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:04,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:04,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:04,957 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:06,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:06,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:07,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:07,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:07,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:08,520 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:08,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:08,958 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:10,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:10,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:10,966 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:12,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:12,800 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:13,061 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:14,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:15,004 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:17,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:17,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:18,062 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:19,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:19,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:19,571 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:06:20,969 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:21,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:21,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:22,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:22,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:23,334 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:23,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:23,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:27,508 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:06:27,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:27,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:28,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:29,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:29,623 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:31,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:31,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:32,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:06:32,130 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:06:32,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:06:32,130 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:06:32,972 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:32,972 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:06:33,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:33,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:34,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:35,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:35,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:37,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:37,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:37,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:37,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:39,874 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:39,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:39,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:40,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:42,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:42,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:44,128 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:44,150 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:44,976 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:45,608 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:46,171 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:46,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:48,204 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:48,219 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:48,977 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:49,573 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:06:50,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:50,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:50,917 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:50,978 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:51,886 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:06:51,887 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:06:51,887 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:06:51,888 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:06:51,978 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:06:52,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:52,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:52,288 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:52,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:52,978 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:55,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:55,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:56,325 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:57,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:57,511 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:01,326 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:01,442 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:07:01,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:01,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:03,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:03,512 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:05,537 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:05,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:06,328 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:07,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:07,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:07,570 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:07,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:09,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:09,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:10,984 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:11,641 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:11,659 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:12,300 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:13,683 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:13,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:15,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:15,750 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:17,781 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:17,795 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:18,088 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:18,986 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:19,576 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:07:20,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:20,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:22,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:22,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:22,938 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:22,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:22,988 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:23,497 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:24,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:24,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:26,989 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:27,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:27,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:28,989 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:29,041 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:07:29,042 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:07:29,042 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:07:29,043 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:07:29,990 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:07:30,190 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:30,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:30,991 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:32,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:32,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:32,992 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:34,004 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:36,112 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:07:36,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:36,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:37,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:37,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:38,683 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:38,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:39,273 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:40,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:40,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:42,827 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:42,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:44,273 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:44,905 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:44,921 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:47,002 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:47,014 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:48,998 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:49,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:49,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:49,580 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:07:49,581 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:51,186 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:51,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:52,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:52,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:53,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:53,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:55,251 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:55,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:55,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:57,000 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:57,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:57,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:59,697 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:59,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:00,879 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:01,846 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:01,874 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:03,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:03,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:03,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:05,728 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:08:05,729 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:08:05,729 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:08:05,730 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:08:06,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:08:06,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:06,146 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:06,730 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:07,003 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:07,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:07,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:10,271 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:08:10,344 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:10,364 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:11,004 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:12,398 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:12,440 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:12,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:13,006 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:14,524 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:14,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:16,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:16,665 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:17,398 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:18,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:18,729 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:19,581 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:08:20,774 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:20,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:22,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:22,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:22,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:22,841 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:23,278 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:24,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:24,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:26,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:26,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:27,010 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:28,525 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:28,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:28,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:31,015 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:31,030 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:33,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:33,072 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:33,526 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:35,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:35,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:37,012 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:37,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:37,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:37,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:37,265 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:39,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:39,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:39,384 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:41,523 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:08:41,525 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:08:41,525 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:08:41,525 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:08:42,014 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:08:43,014 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:43,534 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:08:43,611 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:43,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:44,526 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:45,709 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:45,745 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:47,015 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:47,832 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:47,848 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:49,583 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:08:49,584 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:49,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:49,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:51,022 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:52,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:52,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:52,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:52,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:53,023 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:54,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:54,081 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:55,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:56,121 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:56,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:58,177 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:58,188 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:00,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:00,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:00,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:02,305 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:02,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:03,026 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:04,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:04,388 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:05,901 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:06,429 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:06,443 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:07,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:07,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:08,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:08,506 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:10,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:10,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:11,260 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:12,594 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:12,606 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:14,094 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:09:14,095 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:09:14,095 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:09:14,096 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:09:15,029 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:15,030 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:09:16,329 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:09:16,802 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:16,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:17,096 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:18,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:18,959 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:19,586 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:09:21,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:21,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:22,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:22,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:22,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:23,032 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:23,128 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:23,150 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:25,198 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:25,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:27,258 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:27,353 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:27,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:29,034 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:29,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:29,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:31,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:31,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:32,374 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:33,036 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:33,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:33,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:35,038 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:35,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:35,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:35,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:09:35,999 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:09:36,000 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:09:36,000 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:09:36,038 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:09:37,039 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:37,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:37,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:37,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:37,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:38,243 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:39,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:39,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:41,972 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:41,984 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:43,243 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:44,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:44,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:46,047 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:46,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:48,244 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:49,588 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:09:50,051 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:09:50,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:50,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:52,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:52,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:53,404 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:53,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:54,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:55,044 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:55,485 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:55,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:57,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:57,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:59,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:59,636 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:59,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:01,046 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:01,704 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:01,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:03,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:03,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:04,467 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:05,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:05,815 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:07,048 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:07,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:07,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:07,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:07,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:09,638 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:11,049 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:11,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:11,161 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:12,473 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:10:12,475 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:10:12,475 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:10:12,475 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:10:13,051 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:13,051 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:10:13,212 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:13,234 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:15,052 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:15,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:15,278 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:15,526 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:17,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:17,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:19,591 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:10:20,593 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:20,778 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:20,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:22,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:22,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:22,820 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:22,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:26,339 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:26,771 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:10:26,794 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:26,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:28,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:28,836 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:30,857 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:30,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:31,341 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:32,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:32,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:33,056 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:34,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:34,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:36,949 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:37,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:37,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:38,161 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:38,190 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:40,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:40,252 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:42,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:42,305 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:42,335 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:43,059 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:44,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:44,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:46,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:46,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:47,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:49,061 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:49,593 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:10:49,594 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:49,613 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:51,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:51,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:52,085 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:10:52,086 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:10:52,086 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:10:52,086 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:10:52,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:52,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:53,062 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:53,062 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:10:53,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:53,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:53,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:55,758 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:55,771 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:57,064 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:58,605 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:58,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:59,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:59,065 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:03,145 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:11:03,185 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:03,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:03,606 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:05,278 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:05,302 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:07,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:07,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:07,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:07,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:09,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:09,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:09,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:11,594 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:11,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:13,069 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:13,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:13,773 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:14,478 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:15,857 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:15,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:17,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:18,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:19,479 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:19,595 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:11:20,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:20,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:22,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:22,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:22,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:22,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:23,072 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:24,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:24,522 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:24,613 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:26,590 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:26,622 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:28,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:28,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:29,450 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:11:29,451 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:11:29,451 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:11:29,452 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:11:30,074 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:11:30,452 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:30,746 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:30,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:31,074 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:33,369 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:33,397 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:35,075 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:36,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:37,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:37,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:37,438 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:11:37,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:37,476 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:39,077 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:39,511 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:39,527 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:41,078 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:41,555 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:41,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:41,985 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:43,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:43,651 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:45,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:45,795 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:46,985 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:47,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:47,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:49,081 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:49,598 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:11:50,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:50,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:52,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:52,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:52,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:52,238 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:52,251 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:54,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:54,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:56,403 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:56,439 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:57,238 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:58,479 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:58,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:00,527 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:00,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:00,592 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:00,593 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:12:00,593 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:12:00,593 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:12:01,084 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:01,084 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:12:02,566 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:02,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:02,594 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:03,085 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:04,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:04,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:06,651 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:06,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:07,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:07,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:07,801 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:09,086 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:10,768 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:12:10,824 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:10,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:12,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:12,886 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:13,088 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:13,582 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:14,908 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:14,925 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:16,946 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:16,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:17,090 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:18,609 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:18,991 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:19,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:19,092 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:19,601 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:12:20,527 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:20,528 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:12:20,528 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:12:20,528 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:12:21,093 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:21,094 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:12:22,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:22,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:22,319 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:22,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:23,094 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:24,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:24,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:24,392 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:26,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:26,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:28,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:28,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:29,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:30,515 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:30,525 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:33,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:33,863 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:34,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:35,899 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:35,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:37,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:37,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:37,951 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:37,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:39,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:40,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:40,024 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:41,100 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:42,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:42,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:45,025 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:45,992 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:12:46,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:46,050 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:48,102 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:48,119 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:49,101 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:49,603 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:12:50,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:50,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:50,605 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:52,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:52,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:52,253 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:52,266 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:53,103 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:54,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:54,329 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:55,815 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:56,366 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:56,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:57,104 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:58,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:58,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:58,630 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:58,631 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:12:58,632 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:12:58,632 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:12:59,105 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:59,106 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:13:00,459 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:00,481 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:01,106 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:01,617 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:02,506 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:02,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:04,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:04,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:06,588 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:06,606 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:06,618 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:07,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:07,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:08,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:08,641 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:10,661 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:10,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:12,341 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:12,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:12,728 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:14,750 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:14,762 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:17,110 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:18,320 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:18,864 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:13:18,891 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:18,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:19,605 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:13:20,930 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:20,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:22,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:22,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:22,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:22,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:23,342 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:25,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:25,014 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:27,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:27,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:27,113 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:28,829 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:29,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:29,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:31,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:31,337 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:33,114 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:33,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:33,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:34,643 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:35,456 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:35,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:35,807 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:13:35,809 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:13:35,809 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:13:35,809 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:13:36,115 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:13:37,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:37,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:37,116 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:37,511 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:37,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:39,117 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:39,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:39,580 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:39,726 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:41,623 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:41,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:43,119 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:43,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:43,688 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:44,901 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:45,720 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:45,738 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:47,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:47,795 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:49,607 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:13:50,608 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:51,568 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:13:52,051 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:52,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:52,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:52,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:54,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:54,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:55,123 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:56,333 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:56,367 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:56,373 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:58,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:58,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:00,614 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:00,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:01,373 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:02,704 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:02,737 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:04,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:04,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:05,127 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:06,818 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:06,914 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:06,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:07,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:07,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:08,977 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:09,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:10,043 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:14:10,044 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:14:10,044 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:14:10,045 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:14:10,128 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:14:11,128 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:11,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:11,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:12,045 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:13,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:13,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:15,129 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:15,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:15,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:17,279 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:17,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:17,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:19,131 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:19,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:19,581 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:19,609 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:14:21,133 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:21,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:21,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:22,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:22,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:23,271 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:25,689 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:14:25,740 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:25,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:27,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:27,856 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:28,273 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:29,884 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:29,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:31,136 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:31,921 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:31,942 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:33,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:33,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:34,254 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:36,002 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:36,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:37,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:37,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:38,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:38,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:39,507 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:40,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:40,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:42,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:42,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:42,898 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:14:42,899 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:14:42,899 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:14:42,899 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:14:43,143 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:43,143 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:14:44,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:44,346 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:44,900 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:45,144 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:46,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:46,470 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:48,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:48,599 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:49,612 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:14:50,371 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:50,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:50,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:51,146 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:52,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:52,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:52,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:52,937 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:55,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:55,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:55,585 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:57,148 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:59,121 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:14:59,186 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:59,212 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:01,119 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:01,149 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:01,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:01,274 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:03,150 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:03,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:03,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:03,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:15:03,650 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:15:03,650 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:15:03,651 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:15:04,150 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:15:05,150 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:05,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:05,395 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:06,651 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:07,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:07,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:07,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:07,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:09,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:09,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:11,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:11,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:12,327 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:13,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:13,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:15,641 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:15,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:17,328 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:17,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:17,693 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:19,615 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:15:19,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:19,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:21,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:21,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:22,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:22,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:23,157 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:23,241 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:24,061 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:24,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:26,206 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:26,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:28,241 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:28,319 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:28,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:31,159 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:32,025 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:15:32,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:32,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:33,684 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:34,568 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:34,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:36,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:36,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:37,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:37,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:37,161 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:38,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:38,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:39,074 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:40,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:40,800 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:41,162 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:42,065 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:15:42,067 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:15:42,067 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:15:42,067 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:15:42,162 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:15:42,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:42,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:43,163 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:44,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:45,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:45,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:45,163 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:47,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:47,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:49,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:49,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:49,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:49,617 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:15:51,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:51,388 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:52,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:52,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:53,464 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:53,493 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:54,267 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:55,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:55,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:57,705 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:57,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:59,268 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:59,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:59,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:01,836 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:01,846 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:03,168 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:05,078 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:05,993 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:16:06,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:06,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:07,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:07,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:08,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:08,077 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:10,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:10,153 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:10,893 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:11,171 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:12,344 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:12,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:14,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:14,545 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:16,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:16,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:16,690 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:17,173 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:18,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:18,807 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:19,475 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:16:19,476 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:16:19,477 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:16:19,477 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:16:19,619 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:16:20,174 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:16:20,935 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:20,962 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:21,174 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:21,621 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:22,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:22,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:23,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:23,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:23,175 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:25,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:25,185 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:25,204 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:27,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:27,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:27,419 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:29,427 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:29,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:31,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:31,563 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:32,420 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:33,631 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:33,661 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:35,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:35,721 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:37,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:37,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:38,255 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:39,181 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:39,770 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:16:39,809 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:39,819 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:41,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:41,891 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:43,256 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:43,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:43,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:45,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:46,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:48,118 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:48,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:48,944 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:49,183 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:49,622 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:16:50,302 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:50,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:52,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:52,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:52,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:52,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:53,332 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:16:53,333 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:16:53,334 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:16:53,334 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:16:54,186 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:16:54,334 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:54,576 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:54,603 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:55,186 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:56,704 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:56,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:58,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:58,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:59,187 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:59,670 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:00,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:00,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:03,060 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:03,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:03,189 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:05,152 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:05,180 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:05,191 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:05,339 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:07,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:07,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:07,252 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:07,265 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:09,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:09,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:11,313 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:13,444 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:17:13,484 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:13,496 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:15,194 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:15,603 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:15,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:16,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:17,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:17,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:19,625 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:17:19,783 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:19,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:21,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:21,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:22,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:22,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:23,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:23,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:25,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:25,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:26,161 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:17:26,792 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:17:26,793 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:17:26,793 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:26,794 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:17:27,197 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:27,198 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:17:28,002 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:28,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:29,198 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:30,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:30,066 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:31,795 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:33,252 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:33,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:35,200 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:35,337 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:35,373 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:37,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:37,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:37,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:37,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:37,427 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:39,202 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:39,448 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:39,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:41,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:41,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:42,432 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:43,203 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:43,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:43,555 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:46,864 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:17:46,865 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:17:46,865 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:17:46,866 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:17:47,204 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:47,205 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:17:47,464 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:17:47,497 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:47,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:47,866 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:49,205 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:49,627 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:17:50,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:50,845 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:52,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:52,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:52,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:52,926 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:53,292 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:54,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:55,002 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:57,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:57,099 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:58,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:00,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:00,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:02,631 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:02,662 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:03,209 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:03,938 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:04,753 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:04,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:06,875 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:06,901 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:07,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:18:07,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:18:07,210 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:08,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:08,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:09,281 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:10,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:11,021 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:13,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:13,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:13,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:15,126 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:15,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:15,193 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:15,213 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:17,170 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:17,186 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:19,629 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:18:20,427 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:20,439 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:20,630 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:22,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:18:22,116 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:18:24,344 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:18:24,370 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:24,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:26,264 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:26,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:26,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:28,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:28,456 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:30,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:30,495 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:31,265 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:33,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:33,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:35,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:35,782 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:36,266 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:37,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:18:37,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:18:37,220 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:37,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:37,816 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:39,230 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:39,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:39,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:41,239 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:41,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:41,927 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:42,071 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:42,576 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 21:18:42,577 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 21:18:42,890 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:18:42,938 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:18:42,940 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media +2024-05-13 21:18:43,752 INFO wandb-upload_0:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmp21h4u61y +2024-05-13 21:18:43,789 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/p8xnqiah-media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json +2024-05-13 21:18:44,300 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ahahvwe5-media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav +2024-05-13 21:18:44,412 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep127 - {'id': 'QXJ0aWZhY3Q6ODM0NzY0OTc0', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI0NTA0OA==', 'latestArtifact': None}} +2024-05-13 21:18:44,412 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,412 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json with policy now +2024-05-13 21:18:44,413 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:18:44,413 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:18:44,413 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:18:44,414 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav with policy now +2024-05-13 21:18:44,414 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav with policy now +2024-05-13 21:18:44,414 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav with policy now +2024-05-13 21:18:44,414 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav with policy now +2024-05-13 21:18:44,414 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav with policy now +2024-05-13 21:18:44,415 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,415 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav with policy now +2024-05-13 21:18:44,415 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,415 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav with policy now +2024-05-13 21:18:44,478 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/4qc266nw-media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav +2024-05-13 21:18:44,480 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ewag4pgb-media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav +2024-05-13 21:18:44,509 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/3q9cism8-media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav +2024-05-13 21:18:44,536 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/iqs57vqj-media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav +2024-05-13 21:18:44,541 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/mglkmosw-media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav +2024-05-13 21:18:44,550 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/wte40eui-media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav +2024-05-13 21:18:44,886 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/uvsggul8-media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json +2024-05-13 21:18:45,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:45,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:18:45,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:45,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:45,469 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/1gpdin7d-media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav +2024-05-13 21:18:45,564 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/zjlxcjkp-media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav +2024-05-13 21:18:45,587 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ajegc54h-media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav +2024-05-13 21:18:45,611 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/z5ylnkvx-media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav +2024-05-13 21:18:45,913 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/95d2petp-media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav +2024-05-13 21:18:46,521 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/fkmf1x1x-media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav +2024-05-13 21:18:46,606 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/w8uqvnxv-media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav +2024-05-13 21:18:47,243 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:47,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:47,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:47,475 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:49,416 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:49,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:49,631 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:18:51,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:51,466 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:52,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:18:52,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:18:53,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:53,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:53,586 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:55,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:55,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:58,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:59,452 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:18:59,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:59,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:01,997 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:02,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:03,325 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:04,119 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:04,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:06,226 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:06,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:07,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:19:07,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:19:07,253 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:08,278 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:08,288 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:10,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:10,343 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:11,824 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:12,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:12,404 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:13,642 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:19:13,643 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:19:13,644 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:19:13,644 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:19:14,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:19:14,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:14,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:15,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:16,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:16,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:17,645 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:18,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:18,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:19,633 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:19:20,702 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:20,737 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:21,259 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:22,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:19:22,116 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:19:22,877 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:22,905 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:22,960 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:24,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:25,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:25,261 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:27,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:27,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:27,262 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:28,811 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:29,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:29,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:29,263 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:31,860 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:19:31,886 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:33,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:33,811 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:35,420 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:35,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:37,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:19:37,116 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:19:37,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:37,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:39,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:39,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:39,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:41,862 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:41,896 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:44,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:44,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:44,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:45,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:19:45,177 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:19:45,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:19:45,178 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:19:45,271 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:19:46,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:46,129 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:47,272 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:48,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:48,293 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:49,636 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:19:49,637 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:49,755 WARNING FileStreamThread:1257956 [file_stream.py:request_with_retry():674] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/sanchit-gandhi/parler-speech/qaoje1x9/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/sanchit-gandhi/parler-speech/qaoje1x9/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"train/loss": 4.445040225982666, "train/time": 819.1428015232086, "train/epoch": 0.984251968503937, "train/learning_rate": 4e-05, "_timestamp": 1715627866.8630917, "_runtime": 1497.8430137634277, "_step": 125}']}, 'wandb-history.jsonl': {'offset': 24, 'content': ['{"train/loss": 4.445040225982666, "train/time": 819.1428015232086, "train/epoch": 0.984251968503937, "train/learning_rate": 4e-05, "_timestamp": 1715627866.8630917, "_runtime": 1497.8430137634277, "_step": 125}']}, 'wandb-events.jsonl': {'offset': 50, 'content': ['{"system.proc.memory.availableMB": 406600.59, "system.memory": 21.15, "system.proc.memory.rssMB": 7339.59, "system.proc.memory.percent": 1.42, "system.network.sent": 215219120.0, "system.network.recv": 139749001.07, "system.cpu": 0.79, "system.cpu.0.cpu_percent": 11.47, "system.cpu.1.cpu_percent": 4.03, "system.cpu.2.cpu_percent": 37.67, "system.cpu.3.cpu_percent": 6.23, "system.cpu.4.cpu_percent": 8.89, "system.cpu.5.cpu_percent": 1.69, "system.cpu.6.cpu_percent": 14.29, "system.cpu.7.cpu_percent": 0.23, "system.cpu.8.cpu_percent": 10.91, "system.cpu.9.cpu_percent": 7.67, "system.cpu.10.cpu_percent": 0.27, "system.cpu.11.cpu_percent": 8.63, "system.cpu.12.cpu_percent": 1.03, "system.cpu.13.cpu_percent": 0.03, "system.cpu.14.cpu_percent": 0.0, "system.cpu.15.cpu_percent": 0.2, "system.cpu.16.cpu_percent": 0.03, "system.cpu.17.cpu_percent": 0.17, "system.cpu.18.cpu_percent": 0.03, "system.cpu.19.cpu_percent": 0.13, "system.cpu.20.cpu_percent": 7.62, "system.cpu.21.cpu_percent": 6.03, "system.cpu.22.cpu_percent": 0.43, "system.cpu.23.cpu_percent": 3.47, "system.cpu.24.cpu_percent": 0.07, "system.cpu.25.cpu_percent": 62.72, "system.cpu.26.cpu_percent": 0.23, "system.cpu.27.cpu_percent": 0.1, "system.cpu.28.cpu_percent": 5.06, "system.cpu.29.cpu_percent": 1.37, "system.cpu.30.cpu_percent": 0.1, "system.cpu.31.cpu_percent": 0.17, "system.cpu.32.cpu_percent": 58.22, "system.cpu.33.cpu_percent": 9.39, "system.cpu.34.cpu_percent": 1.1, "system.cpu.35.cpu_percent": 0.03, "system.cpu.36.cpu_percent": 5.53, "system.cpu.37.cpu_percent": 4.13, "system.cpu.38.cpu_percent": 0.47, "system.cpu.39.cpu_percent": 0.23, "system.cpu.40.cpu_percent": 15.57, "system.cpu.41.cpu_percent": 8.1, "system.cpu.42.cpu_percent": 0.7, "system.cpu.43.cpu_percent": 0.63, "system.cpu.44.cpu_percent": 2.65, "system.cpu.45.cpu_percent": 8.46, "system.cpu.46.cpu_percent": 2.2, "system.cpu.47.cpu_percent": 1.45, "system.cpu.48.cpu_percent": 44.57, "system.cpu.49.cpu_percent": 0.03, "system.cpu.50.cpu_percent": 34.19, "system.cpu.51.cpu_percent": 26.78, "system.cpu.52.cpu_percent": 20.47, "system.cpu.53.cpu_percent": 22.31, "system.cpu.54.cpu_percent": 2.37, "system.cpu.55.cpu_percent": 1.59, "system.cpu.56.cpu_percent": 0.87, "system.cpu.57.cpu_percent": 31.11, "system.cpu.58.cpu_percent": 0.73, "system.cpu.59.cpu_percent": 3.45, "system.cpu.60.cpu_percent": 7.25, "system.cpu.61.cpu_percent": 0.07, "system.cpu.62.cpu_percent": 0.1, "system.cpu.63.cpu_percent": 2.47, "system.cpu.64.cpu_percent": 0.13, "system.cpu.65.cpu_percent": 3.37, "system.cpu.66.cpu_percent": 8.77, "system.cpu.67.cpu_percent": 3.01, "system.cpu.68.cpu_percent": 0.57, "system.cpu.69.cpu_percent": 1.84, "system.cpu.70.cpu_percent": 2.23, "system.cpu.71.cpu_percent": 0.87, "system.cpu.72.cpu_percent": 0.4, "system.cpu.73.cpu_percent": 12.17, "system.cpu.74.cpu_percent": 1.23, "system.cpu.75.cpu_percent": 1.69, "system.cpu.76.cpu_percent": 0.23, "system.cpu.77.cpu_percent": 0.33, "system.cpu.78.cpu_percent": 0.13, "system.cpu.79.cpu_percent": 0.2, "system.cpu.80.cpu_percent": 99.53, "system.cpu.81.cpu_percent": 0.1, "system.cpu.82.cpu_percent": 0.07, "system.cpu.83.cpu_percent": 0.07, "system.cpu.84.cpu_percent": 0.0, "system.cpu.85.cpu_percent": 1.6, "system.cpu.86.cpu_percent": 0.1, "system.cpu.87.cpu_percent": 0.07, "system.cpu.88.cpu_percent": 36.33, "system.cpu.89.cpu_percent": 0.13, "system.cpu.90.cpu_percent": 0.1, "system.cpu.91.cpu_percent": 0.17, "system.cpu.92.cpu_percent": 0.3, "system.cpu.93.cpu_percent": 4.84, "system.cpu.94.cpu_percent": 0.87, "system.cpu.95.cpu_percent": 1.76, "system.cpu.96.cpu_percent": 0.2, "system.cpu.97.cpu_percent": 0.07, "system.cpu.98.cpu_percent": 0.3, "system.cpu.99.cpu_percent": 0.2, "system.cpu.100.cpu_percent": 3.26, "system.cpu.101.cpu_percent": 2.1, "system.cpu.102.cpu_percent": 0.33, "system.cpu.103.cpu_percent": 0.64, "system.cpu.104.cpu_percent": 2.16, "system.cpu.105.cpu_percent": 4.71, "system.cpu.106.cpu_percent": 0.8, "system.cpu.107.cpu_percent": 0.03, "system.cpu.108.cpu_percent": 0.13, "system.cpu.109.cpu_percent": 0.5, "system.cpu.110.cpu_percent": 0.23, "system.cpu.111.cpu_percent": 2.47, "system.cpu.112.cpu_percent": 0.13, "system.cpu.113.cpu_percent": 0.03, "system.cpu.114.cpu_percent": 0.1, "system.cpu.115.cpu_percent": 0.0, "system.cpu.116.cpu_percent": 0.23, "system.cpu.117.cpu_percent": 0.03, "system.cpu.118.cpu_percent": 3.53, "system.cpu.119.cpu_percent": 0.1, "system.cpu.120.cpu_percent": 0.8, "system.cpu.121.cpu_percent": 1.03, "system.cpu.122.cpu_percent": 0.8, "system.cpu.123.cpu_percent": 0.57, "system.cpu.124.cpu_percent": 5.35, "system.cpu.125.cpu_percent": 0.37, "system.cpu.126.cpu_percent": 0.13, "system.cpu.127.cpu_percent": 0.1, "system.proc.cpu.threads": 78, "system.disk.\\\\.usagePercent": 99.7, "system.disk.\\\\.usageGB": 1663.51, "system.disk.in": 65.27, "system.disk.out": 13959.2, "system.gpu.0.memoryAllocated": 85.43, "system.gpu.1.memoryAllocated": 68.86, "system.gpu.process.1.memoryAllocated": 68.86, "system.gpu.2.memoryAllocated": 59.42, "system.gpu.3.memoryAllocated": 4.69, "system.gpu.4.memoryAllocated": 42.38, "system.gpu.0.memoryAllocatedBytes": 73387802624.0, "system.gpu.1.memoryAllocatedBytes": 59154497536.0, "system.gpu.process.1.memoryAllocatedBytes": 59154497536.0, "system.gpu.2.memoryAllocatedBytes": 51038388224.0, "system.gpu.3.memoryAllocatedBytes": 201588736.0, "system.gpu.4.memoryAllocatedBytes": 36444866696.53, "system.gpu.0.memory": 0.0, "system.gpu.1.memory": 30.2, "system.gpu.process.1.memory": 30.2, "system.gpu.2.memory": 48.13, "system.gpu.3.memory": 0.0, "system.gpu.4.memory": 34.67, "system.gpu.0.gpu": 0.0, "system.gpu.1.gpu": 66.8, "system.gpu.process.1.gpu": 66.8, "system.gpu.2.gpu": 96.87, "system.gpu.3.gpu": 0.0, "system.gpu.4.gpu": 74.93, "system.gpu.0.temp": 70.87, "system.gpu.1.temp": 77.07, "system.gpu.process.1.temp": 77.07, "system.gpu.2.temp": 77.73, "system.gpu.3.temp": 40.0, "system.gpu.4.temp": 77.13, "_wandb": true, "_timestamp": 1715627929.628074, "_runtime": 1560.607996}']}, 'output.log': {'offset': 921, 'content': ['ERROR 2024-05-13T19:18:39.230068 31,7,r_norm_epsilon": 1e-06,\n', 'ERROR 2024-05-13T19:18:39.230068 3246,\n', 'ERROR 2024-05-13T19:18:39.230068 3253,\n', 'ERROR 2024-05-13T19:18:39.230068 3268,\n', 'ERROR 2024-05-13T19:18:39.230068 3536,\n', 'ERROR 2024-05-13T19:18:39.230068 3846,\n', 'ERROR 2024-05-13T19:18:39.230068 3961,\n', 'ERROR 2024-05-13T19:18:39.230068 4183,\n', 'ERROR 2024-05-13T19:18:39.230068 4667,\n', 'ERROR 2024-05-13T19:18:39.230068 6585,\n', 'ERROR 2024-05-13T19:18:39.230068 6647,\n', 'ERROR 2024-05-13T19:18:39.230068 7273,\n', 'ERROR 2024-05-13T19:18:39.230068 9061,\n', 'ERROR 2024-05-13T19:18:39.230068 9383,\n', 'ERROR 2024-05-13T19:18:39.230068 10428,\n', 'ERROR 2024-05-13T19:18:39.230068 10929,\n', 'ERROR 2024-05-13T19:18:39.230068 11938,\n', 'ERROR 2024-05-13T19:18:39.230068 12033,\n', 'ERROR 2024-05-13T19:18:39.230068 12331,\n', 'ERROR 2024-05-13T19:18:39.230068 12562,\n', 'ERROR 2024-05-13T19:18:39.230068 13793,\n', 'ERROR 2024-05-13T19:18:39.230068 14157,\n', 'ERROR 2024-05-13T19:18:39.230068 14635,\n', 'ERROR 2024-05-13T19:18:39.230068 15265,\n', 'ERROR 2024-05-13T19:18:39.230068 15618,\n', 'ERROR 2024-05-13T19:18:39.230068 16553,\n', 'ERROR 2024-05-13T19:18:39.230068 16604,\n', 'ERROR 2024-05-13T19:18:39.230068 18362,\n', 'ERROR 2024-05-13T19:18:39.230068 18956,\n', 'ERROR 2024-05-13T19:18:39.230068 20075,\n', 'ERROR 2024-05-13T19:18:39.230068 21675,\n', 'ERROR 2024-05-13T19:18:39.230068 22520,\n', 'ERROR 2024-05-13T19:18:39.230068 26130,\n', 'ERROR 2024-05-13T19:18:39.230068 26161,\n', 'ERROR 2024-05-13T19:18:39.230068 26435,\n', 'ERROR 2024-05-13T19:18:39.230068 28279,\n', 'ERROR 2024-05-13T19:18:39.230068 29464,\n', 'ERROR 2024-05-13T19:18:39.230068 31650,\n', 'ERROR 2024-05-13T19:18:39.230068 32302,\n', 'ERROR 2024-05-13T19:18:39.230068 32470,\n', 'ERROR 2024-05-13T19:18:39.230068 36865,\n', 'ERROR 2024-05-13T19:18:39.230068 42863,\n', 'ERROR 2024-05-13T19:18:39.230068 47425,\n', 'ERROR 2024-05-13T19:18:39.230068 49870,\n', 'ERROR 2024-05-13T19:18:39.230068 50254,\n', 'ERROR 2024-05-13T19:18:39.230068 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 50360,\n', 'ERROR 2024-05-13T19:18:39.230068 50361,\n', 'ERROR 2024-05-13T19:18:39.230068 50362\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "torch_dtype": "float32",\n', 'ERROR 2024-05-13T19:18:39.230068 "transformers_version": "4.41.0.dev0",\n', 'ERROR 2024-05-13T19:18:39.230068 "use_cache": true,\n', 'ERROR 2024-05-13T19:18:39.230068 "use_weighted_layer_sum": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "vocab_size": 51865\n', 'ERROR 2024-05-13T19:18:39.230068 }\n', 'ERROR 2024-05-13T19:18:39.230068 loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json\n', 'ERROR 2024-05-13T19:18:39.230068 Model config WhisperConfig {\n', 'ERROR 2024-05-13T19:18:39.230068 "_name_or_path": "distil-whisper/distil-large-v2",\n', 'ERROR 2024-05-13T19:18:39.230068 "activation_dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "activation_function": "gelu",\n', 'ERROR 2024-05-13T19:18:39.230068 "apply_spec_augment": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "architectures": [\n', 'ERROR 2024-05-13T19:18:39.230068 "WhisperForConditionalGeneration"\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "attention_dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "begin_suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 220,\n', 'ERROR 2024-05-13T19:18:39.230068 50257\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "bos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "classifier_proj_size": 256,\n', 'ERROR 2024-05-13T19:18:39.230068 "d_model": 1280,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_attention_heads": 20,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_ffn_dim": 5120,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_layerdrop": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_layers": 2,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_start_token_id": 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 "dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_attention_heads": 20,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_ffn_dim": 5120,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_layerdrop": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_layers": 32,\n', 'ERROR 2024-05-13T19:18:39.230068 "eos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "forced_decoder_ids": [\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 50259\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 50359\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 3,\n', 'ERROR 2024-05-13T19:18:39.230068 50363\n', 'ERROR 2024-05-13T19:18:39.230068 ]\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "init_std": 0.02,\n', 'ERROR 2024-05-13T19:18:39.230068 "is_encoder_decoder": true,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_length": 10,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_min_masks": 0,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_prob": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_length": 10,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_min_masks": 2,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_prob": 0.05,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_length": 448,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_source_positions": 1500,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_target_positions": 448,\n', 'ERROR 2024-05-13T19:18:39.230068 "median_filter_width": 7,\n', 'ERROR 2024-05-13T19:18:39.230068 "model_type": "whisper",\n', 'ERROR 2024-05-13T19:18:39.230068 "num_hidden_layers": 32,\n', 'ERROR 2024-05-13T19:18:39.230068 "num_mel_bins": 80,\n', 'ERROR 2024-05-13T19:18:39.230068 "pad_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "scale_embedding": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 7,\n', 'ERROR 2024-05-13T19:18:39.230068 8,\n', 'ERROR 2024-05-13T19:18:39.230068 9,\n', 'ERROR 2024-05-13T19:18:39.230068 10,\n', 'ERROR 2024-05-13T19:18:39.230068 14,\n', 'ERROR 2024-05-13T19:18:39.230068 25,\n', 'ERROR 2024-05-13T19:18:39.230068 26,\n', 'ERROR 2024-05-13T19:18:39.230068 27,\n', 'ERROR 2024-05-13T19:18:39.230068 28,\n', 'ERROR 2024-05-13T19:18:39.230068 29,\n', 'ERROR 2024-05-13T19:18:39.230068 31,7,r_norm_epsilon": 1e-06,\n', 'ERROR 2024-05-13T19:18:39.230068 58,\n', 'ERROR 2024-05-13T19:18:39.230068 59,\n', 'ERROR 2024-05-13T19:18:39.230068 60,\n', 'ERROR 2024-05-13T19:18:39.230068 61,\n', 'ERROR 2024-05-13T19:18:39.230068 62,\n', 'ERROR 2024-05-13T19:18:39.230068 63,\n', 'ERROR 2024-05-13T19:18:39.230068 90,\n', 'ERROR 2024-05-13T19:18:39.230068 91,\n', 'ERROR 2024-05-13T19:18:39.230068 92,\n', 'ERROR 2024-05-13T19:18:39.230068 93,\n', 'ERROR 2024-05-13T19:18:39.230068 359,\n', 'ERROR 2024-05-13T19:18:39.230068 503,\n', 'ERROR 2024-05-13T19:18:39.230068 522,\n', 'ERROR 2024-05-13T19:18:39.230068 542,\n', 'ERROR 2024-05-13T19:18:39.230068 873,\n', 'ERROR 2024-05-13T19:18:39.230068 893,\n', 'ERROR 2024-05-13T19:18:39.230068 902,\n', 'ERROR 2024-05-13T19:18:39.230068 918,\n', 'ERROR 2024-05-13T19:18:39.230068 922,\n', 'ERROR 2024-05-13T19:18:39.230068 931,\n', 'ERROR 2024-05-13T19:18:39.230068 1350,\n', 'ERROR 2024-05-13T19:18:39.230068 1853,\n', 'ERROR 2024-05-13T19:18:39.230068 1982,\n', 'ERROR 2024-05-13T19:18:39.230068 2460,\n', 'ERROR 2024-05-13T19:18:39.230068 2627,\n', 'ERROR 2024-05-13T19:18:39.230068 3246,\n', 'ERROR 2024-05-13T19:18:39.230068 3253,\n', 'ERROR 2024-05-13T19:18:39.230068 3268,\n', 'ERROR 2024-05-13T19:18:39.230068 3536,\n', 'ERROR 2024-05-13T19:18:39.230068 3846,\n', 'ERROR 2024-05-13T19:18:39.230068 3961,\n', 'ERROR 2024-05-13T19:18:39.230068 4183,\n', 'ERROR 2024-05-13T19:18:39.230068 4667,\n', 'ERROR 2024-05-13T19:18:39.230068 6585,\n', 'ERROR 2024-05-13T19:18:39.230068 6647,\n', 'ERROR 2024-05-13T19:18:39.230068 7273,\n', 'ERROR 2024-05-13T19:18:39.230068 9061,\n', 'ERROR 2024-05-13T19:18:39.230068 9383,\n', 'ERROR 2024-05-13T19:18:39.230068 10428,\n', 'ERROR 2024-05-13T19:18:39.230068 10929,\n', 'ERROR 2024-05-13T19:18:39.230068 11938,\n', 'ERROR 2024-05-13T19:18:39.230068 12033,\n', 'ERROR 2024-05-13T19:18:39.230068 12331,\n', 'ERROR 2024-05-13T19:18:39.230068 12562,\n', 'ERROR 2024-05-13T19:18:39.230068 13793,\n', 'ERROR 2024-05-13T19:18:39.230068 14157,\n', 'ERROR 2024-05-13T19:18:39.230068 14635,\n', 'ERROR 2024-05-13T19:18:39.230068 15265,\n', 'ERROR 2024-05-13T19:18:39.230068 15618,\n', 'ERROR 2024-05-13T19:18:39.230068 16553,\n', 'ERROR 2024-05-13T19:18:39.230068 16604,\n', 'ERROR 2024-05-13T19:18:39.230068 18362,\n', 'ERROR 2024-05-13T19:18:39.230068 18956,\n', 'ERROR 2024-05-13T19:18:39.230068 20075,\n', 'ERROR 2024-05-13T19:18:39.230068 21675,\n', 'ERROR 2024-05-13T19:18:39.230068 22520,\n', 'ERROR 2024-05-13T19:18:39.230068 26130,\n', 'ERROR 2024-05-13T19:18:39.230068 26161,\n', 'ERROR 2024-05-13T19:18:39.230068 26435,\n', 'ERROR 2024-05-13T19:18:39.230068 28279,\n', 'ERROR 2024-05-13T19:18:39.230068 29464,\n', 'ERROR 2024-05-13T19:18:39.230068 31650,\n', 'ERROR 2024-05-13T19:18:39.230068 32302,\n', 'ERROR 2024-05-13T19:18:39.230068 32470,\n', 'ERROR 2024-05-13T19:18:39.230068 36865,\n', 'ERROR 2024-05-13T19:18:39.230068 42863,\n', 'ERROR 2024-05-13T19:18:39.230068 47425,\n', 'ERROR 2024-05-13T19:18:39.230068 49870,\n', 'ERROR 2024-05-13T19:18:39.230068 50254,\n', 'ERROR 2024-05-13T19:18:39.230068 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 50360,\n', 'ERROR 2024-05-13T19:18:39.230068 50361,\n', 'ERROR 2024-05-13T19:18:39.230068 50362\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "torch_dtype": "float32",\n', 'ERROR 2024-05-13T19:18:39.230068 "transformers_version": "4.41.0.dev0",\n', 'ERROR 2024-05-13T19:18:39.230068 "use_cache": true,\n', 'ERROR 2024-05-13T19:18:39.230068 "use_weighted_layer_sum": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "vocab_size": 51865\n', 'ERROR 2024-05-13T19:18:39.230068 }\n', 'ERROR 2024-05-13T19:18:39.230068 loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json\n', 'ERROR 2024-05-13T19:18:39.230068 Model config WhisperConfig {\n', 'ERROR 2024-05-13T19:18:39.230068 "_name_or_path": "distil-whisper/distil-large-v2",\n', 'ERROR 2024-05-13T19:18:39.230068 "activation_dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "activation_function": "gelu",\n', 'ERROR 2024-05-13T19:18:39.230068 "apply_spec_augment": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "architectures": [\n', 'ERROR 2024-05-13T19:18:39.230068 "WhisperForConditionalGeneration"\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "attention_dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "begin_suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 220,\n', 'ERROR 2024-05-13T19:18:39.230068 50257\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "bos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "classifier_proj_size": 256,\n', 'ERROR 2024-05-13T19:18:39.230068 "d_model": 1280,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_attention_heads": 20,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_ffn_dim": 5120,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_layerdrop": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_layers": 2,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_start_token_id": 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 "dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_attention_heads": 20,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_ffn_dim": 5120,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_layerdrop": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_layers": 32,\n', 'ERROR 2024-05-13T19:18:39.230068 "eos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "forced_decoder_ids": [\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 50259\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 50359\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 3,\n', 'ERROR 2024-05-13T19:18:39.230068 50363\n', 'ERROR 2024-05-13T19:18:39.230068 ]\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "init_std": 0.02,\n', 'ERROR 2024-05-13T19:18:39.230068 "is_encoder_decoder": true,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_length": 10,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_min_masks": 0,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_prob": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_length": 10,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_min_masks": 2,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_prob": 0.05,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_length": 448,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_source_positions": 1500,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_target_positions": 448,\n', 'ERROR 2024-05-13T19:18:39.230068 "median_filter_width": 7,\n', 'ERROR 2024-05-13T19:18:39.230068 "model_type": "whisper",\n', 'ERROR 2024-05-13T19:18:39.230068 "num_hidden_layers": 32,\n', 'ERROR 2024-05-13T19:18:39.230068 "num_mel_bins": 80,\n', 'ERROR 2024-05-13T19:18:39.230068 "pad_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "scale_embedding": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 7,\n', 'ERROR 2024-05-13T19:18:39.230068 8,\n', 'ERROR 2024-05-13T19:18:39.230068 9,\n', 'ERROR 2024-05-13T19:18:39.230068 10,\n', 'ERROR 2024-05-13T19:18:39.230068 14,\n', 'ERROR 2024-05-13T19:18:39.230068 25,\n', 'ERROR 2024-05-13T19:18:39.230068 26,\n', 'ERROR 2024-05-13T19:18:39.230068 27,\n', 'ERROR 2024-05-13T19:18:39.230068 28,\n', 'ERROR 2024-05-13T19:18:39.230068 29,\n', 'ERROR 2024-05-13T19:18:39.230068 31,\n', 'ERROR 2024-05-13T19:18:39.230068 58,\n', 'ERROR 2024-05-13T19:18:39.230068 59,\n', 'ERROR 2024-05-13T19:18:39.230068 60,\n', 'ERROR 2024-05-13T19:18:39.230068 61,\n', 'ERROR 2024-05-13T19:18:39.230068 62,\n', 'ERROR 2024-05-13T19:18:39.230068 63,\n', 'ERROR 2024-05-13T19:18:39.230068 90,\n', 'ERROR 2024-05-13T19:18:39.230068 91,\n', 'ERROR 2024-05-13T19:18:39.230068 92,\n', 'ERROR 2024-05-13T19:18:39.230068 93,\n', 'ERROR 2024-05-13T19:18:39.230068 359,\n', 'ERROR 2024-05-13T19:18:39.230068 503,\n', 'ERROR 2024-05-13T19:18:39.230068 522,\n', 'ERROR 2024-05-13T19:18:39.230068 542,\n', 'ERROR 2024-05-13T19:18:39.230068 873,\n', 'ERROR 2024-05-13T19:18:39.230068 893,\n', 'ERROR 2024-05-13T19:18:39.230068 902,\n', 'ERROR 2024-05-13T19:18:39.230068 918,\n', 'ERROR 2024-05-13T19:18:39.230068 922,\n', 'ERROR 2024-05-13T19:18:39.230068 931,\n', 'ERROR 2024-05-13T19:18:39.230068 1350,\n', 'ERROR 2024-05-13T19:18:39.230068 1853,\n', 'ERROR 2024-05-13T19:18:39.230068 1982,\n', 'ERROR 2024-05-13T19:18:39.230068 2460,\n', 'ERROR 2024-05-13T19:18:39.230068 2627,\n', 'ERROR 2024-05-13T19:18:39.230068 3246,\n', 'ERROR 2024-05-13T19:18:39.230068 3253,\n', 'ERROR 2024-05-13T19:18:39.230068 3268,\n', 'ERROR 2024-05-13T19:18:39.230068 3536,\n', 'ERROR 2024-05-13T19:18:39.230068 3846,\n', 'ERROR 2024-05-13T19:18:39.230068 3961,\n', 'ERROR 2024-05-13T19:18:39.230068 4183,\n', 'ERROR 2024-05-13T19:18:39.230068 4667,\n', 'ERROR 2024-05-13T19:18:39.230068 6585,\n', 'ERROR 2024-05-13T19:18:39.230068 6647,\n', 'ERROR 2024-05-13T19:18:39.230068 7273,\n', 'ERROR 2024-05-13T19:18:39.230068 9061,\n', 'ERROR 2024-05-13T19:18:39.230068 9383,\n', 'ERROR 2024-05-13T19:18:39.230068 10428,\n', 'ERROR 2024-05-13T19:18:39.230068 10929,\n', 'ERROR 2024-05-13T19:18:39.230068 11938,\n', 'ERROR 2024-05-13T19:18:39.230068 12033,\n', 'ERROR 2024-05-13T19:18:39.230068 12331,\n', 'ERROR 2024-05-13T19:18:39.230068 12562,\n', 'ERROR 2024-05-13T19:18:39.230068 13793,\n', 'ERROR 2024-05-13T19:18:39.230068 14157,\n', 'ERROR 2024-05-13T19:18:39.230068 14635,\n', 'ERROR 2024-05-13T19:18:39.230068 15265,\n', 'ERROR 2024-05-13T19:18:39.230068 15618,\n', 'ERROR 2024-05-13T19:18:39.230068 16553,\n', 'ERROR 2024-05-13T19:18:39.230068 16604,\n', 'ERROR 2024-05-13T19:18:39.230068 18362,\n', 'ERROR 2024-05-13T19:18:39.230068 18956,\n', 'ERROR 2024-05-13T19:18:39.230068 20075,\n', 'ERROR 2024-05-13T19:18:39.230068 21675,\n', 'ERROR 2024-05-13T19:18:39.230068 22520,\n', 'ERROR 2024-05-13T19:18:39.230068 26130,\n', 'ERROR 2024-05-13T19:18:39.230068 26161,\n', 'ERROR 2024-05-13T19:18:39.230068 26435,\n', 'ERROR 2024-05-13T19:18:39.230068 28279,\n', 'ERROR 2024-05-13T19:18:39.230068 29464,\n', 'ERROR 2024-05-13T19:18:39.230068 31650,\n', 'ERROR 2024-05-13T19:18:39.230068 32302,\n', 'ERROR 2024-05-13T19:18:39.230068 32470,\n', 'ERROR 2024-05-13T19:18:39.230068 36865,\n', 'ERROR 2024-05-13T19:18:39.230068 42863,\n', 'ERROR 2024-05-13T19:18:39.230068 47425,\n', 'ERROR 2024-05-13T19:18:39.230068 49870,\n', 'ERROR 2024-05-13T19:18:39.230068 50254,\n', 'ERROR 2024-05-13T19:18:39.230068 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 50360,\n', 'ERROR 2024-05-13T19:18:39.230068 50361,\n', 'ERROR 2024-05-13T19:18:39.230068 50362\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "torch_dtype": "float32",\n', 'ERROR 2024-05-13T19:18:39.230068 "transformers_version": "4.41.0.dev0",\n', 'ERROR 2024-05-13T19:18:39.230068 "use_cache": true,\n', 'ERROR 2024-05-13T19:18:39.230068 "use_weighted_layer_sum": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "vocab_size": 51865\n', 'ERROR 2024-05-13T19:18:39.230068 }\n', 'ERROR 2024-05-13T19:18:39.230068 loading weights file model.safetensors from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/model.safetensors\n', 'ERROR 2024-05-13T19:18:39.230068 Generate config GenerationConfig {\n', 'ERROR 2024-05-13T19:18:39.230068 "begin_suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 220,\n', 'ERROR 2024-05-13T19:18:39.230068 50257\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "bos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_start_token_id": 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 "eos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "forced_decoder_ids": [\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 50259\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 50359\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 3,\n', 'ERROR 2024-05-13T19:18:39.230068 50363\n', 'ERROR 2024-05-13T19:18:39.230068 ]\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "max_length": 448,\n', 'ERROR 2024-05-13T19:18:39.230068 "pad_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 7,\n', 'ERROR 2024-05-13T19:18:39.230068 8,\n', 'ERROR 2024-05-13T19:18:39.230068 9,\n', 'ERROR 2024-05-13T19:18:39.230068 10,\n', 'ERROR 2024-05-13T19:18:39.230068 14,\n', 'ERROR 2024-05-13T19:18:39.230068 25,\n', 'ERROR 2024-05-13T19:18:39.230068 26,\n', 'ERROR 2024-05-13T19:18:39.230068 27,\n', 'ERROR 2024-05-13T19:18:39.230068 28,\n', 'ERROR 2024-05-13T19:18:39.230068 29,\n', 'ERROR 2024-05-13T19:18:39.230068 31,\n', 'ERROR 2024-05-13T19:18:39.230068 58,\n', 'ERROR 2024-05-13T19:18:39.230068 59,\n', 'ERROR 2024-05-13T19:18:39.230068 60,\n', 'ERROR 2024-05-13T19:18:39.230068 61,\n', 'ERROR 2024-05-13T19:18:39.230068 62,\n', 'ERROR 2024-05-13T19:18:39.230068 63,\n', 'ERROR 2024-05-13T19:18:39.230068 90,\n', 'ERROR 2024-05-13T19:18:39.230068 91,\n', 'ERROR 2024-05-13T19:18:39.230068 92,\n', 'ERROR 2024-05-13T19:18:39.230068 93,\n', 'ERROR 2024-05-13T19:18:39.230068 359,\n', 'ERROR 2024-05-13T19:18:39.230068 503,\n', 'ERROR 2024-05-13T19:18:39.230068 522,\n', 'ERROR 2024-05-13T19:18:39.230068 542,\n', 'ERROR 2024-05-13T19:18:39.230068 873,\n', 'ERROR 2024-05-13T19:18:39.230068 893,\n', 'ERROR 2024-05-13T19:18:39.230068 902,\n', 'ERROR 2024-05-13T19:18:39.230068 918,\n', 'ERROR 2024-05-13T19:18:39.230068 922,\n', 'ERROR 2024-05-13T19:18:39.230068 931,\n', 'ERROR 2024-05-13T19:18:39.230068 1350,\n', 'ERROR 2024-05-13T19:18:39.230068 1853,\n', 'ERROR 2024-05-13T19:18:39.230068 1982,\n', 'ERROR 2024-05-13T19:18:39.230068 2460,\n', 'ERROR 2024-05-13T19:18:39.230068 2627,\n', 'ERROR 2024-05-13T19:18:39.230068 3246,\n', 'ERROR 2024-05-13T19:18:39.230068 3253,\n', 'ERROR 2024-05-13T19:18:39.230068 3268,\n', 'ERROR 2024-05-13T19:18:39.230068 3536,\n', 'ERROR 2024-05-13T19:18:39.230068 3846,\n', 'ERROR 2024-05-13T19:18:39.230068 3961,\n', 'ERROR 2024-05-13T19:18:39.230068 4183,\n', 'ERROR 2024-05-13T19:18:39.230068 4667,\n', 'ERROR 2024-05-13T19:18:39.230068 6585,\n', 'ERROR 2024-05-13T19:18:39.230068 6647,\n', 'ERROR 2024-05-13T19:18:39.230068 7273,\n', 'ERROR 2024-05-13T19:18:39.230068 9061,\n', 'ERROR 2024-05-13T19:18:39.230068 9383,\n', 'ERROR 2024-05-13T19:18:39.230068 10428,\n', 'ERROR 2024-05-13T19:18:39.230068 10929,\n', 'ERROR 2024-05-13T19:18:39.230068 11938,\n', 'ERROR 2024-05-13T19:18:39.230068 12033,\n', 'ERROR 2024-05-13T19:18:39.230068 12331,\n', 'ERROR 2024-05-13T19:18:39.230068 12562,\n', 'ERROR 2024-05-13T19:18:39.230068 13793,\n', 'ERROR 2024-05-13T19:18:39.230068 14157,\n', 'ERROR 2024-05-13T19:18:39.230068 14635,\n', 'ERROR 2024-05-13T19:18:39.230068 15265,\n', 'ERROR 2024-05-13T19:18:39.230068 15618,\n', 'ERROR 2024-05-13T19:18:39.230068 16553,\n', 'ERROR 2024-05-13T19:18:39.230068 16604,\n', 'ERROR 2024-05-13T19:18:39.230068 18362,\n', 'ERROR 2024-05-13T19:18:39.230068 18956,\n', 'ERROR 2024-05-13T19:18:39.230068 20075,\n', 'ERROR 2024-05-13T19:18:39.230068 21675,\n', 'ERROR 2024-05-13T19:18:39.230068 22520,\n', 'ERROR 2024-05-13T19:18:39.230068 26130,\n', 'ERROR 2024-05-13T19:18:39.230068 26161,\n', 'ERROR 2024-05-13T19:18:39.230068 26435,\n', 'ERROR 2024-05-13T19:18:39.230068 28279,\n', 'ERROR 2024-05-13T19:18:39.230068 29464,\n', 'ERROR 2024-05-13T19:18:39.230068 31650,\n', 'ERROR 2024-05-13T19:18:39.230068 32302,\n', 'ERROR 2024-05-13T19:18:39.230068 32470,\n', 'ERROR 2024-05-13T19:18:39.230068 36865,\n', 'ERROR 2024-05-13T19:18:39.230068 42863,\n', 'ERROR 2024-05-13T19:18:39.230068 47425,\n', 'ERROR 2024-05-13T19:18:39.230068 49870,\n', 'ERROR 2024-05-13T19:18:39.230068 50254,\n', 'ERROR 2024-05-13T19:18:39.230068 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 50360,\n', 'ERROR 2024-05-13T19:18:39.230068 50361,\n', 'ERROR 2024-05-13T19:18:39.230068 50362\n', 'ERROR 2024-05-13T19:18:39.230068 ]\n', 'ERROR 2024-05-13T19:18:39.230068 }\n', 'ERROR 2024-05-13T19:18:41.238986 All model checkpoint weights were used when initializing WhisperForConditionalGeneration.\n', 'ERROR 2024-05-13T19:18:41.238986 All the weights of WhisperForConditionalGeneration were initialized from the model checkpoint at distil-whisper/distil-large-v2.\n', 'ERROR 2024-05-13T19:18:41.238986 If your task is similar to the task the model of the checkpoint was trained on, you can already use WhisperForConditionalGeneration for predictions without further training.\n', 'ERROR 2024-05-13T19:18:41.238986 loading configuration file generation_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/generation_config.json\n', 'ERROR 2024-05-13T19:18:41.238986 Generate config GenerationConfig {\n', 'ERROR 2024-05-13T19:18:41.238986 "begin_suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:41.238986 220,\n', 'ERROR 2024-05-13T19:18:41.238986 50257\n', 'ERROR 2024-05-13T19:18:41.238986 ],\n', 'ERROR 2024-05-13T19:18:41.238986 "bos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:41.238986 "decoder_start_token_id": 50258,\n', 'ERROR 2024-05-13T19:18:41.238986 "eos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:41.238986 "forced_decoder_ids": [\n', 'ERROR 2024-05-13T19:18:41.238986 [\n', 'ERROR 2024-05-13T19:18:41.238986 1,\n', 'ERROR 2024-05-13T19:18:41.238986 50259\n', 'ERROR 2024-05-13T19:18:41.238986 ],\n', 'ERROR 2024-05-13T19:18:41.238986 [\n', 'ERROR 2024-05-13T19:18:41.238986 2,\n', 'ERROR 2024-05-13T19:18:41.238986 50359\n', 'ERROR 2024-05-13T19:18:41.238986 ],\n', 'ERROR 2024-05-13T19:18:41.238986 [\n', 'ERROR 2024-05-13T19:18:41.238986 3,\n', 'ERROR 2024-05-13T19:18:41.238986 50363\n', 'ERROR 2024-05-13T19:18:41.238986 ]\n', 'ERROR 2024-05-13T19:18:41.238986 ],\n', 'ERROR 2024-05-13T19:18:41.238986 "is_multilingual": true,\n', 'ERROR 2024-05-13T19:18:41.238986 "lang_to_id": {\n', 'ERROR 2024-05-13T19:18:41.238986 "<|af|>": 50327,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|am|>": 50334,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ar|>": 50272,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|as|>": 50350,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|az|>": 50304,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ba|>": 50355,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|be|>": 50330,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|bg|>": 50292,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|bn|>": 50302,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|bo|>": 50347,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|br|>": 50309,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|bs|>": 50315,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ca|>": 50270,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|cs|>": 50283,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|cy|>": 50297,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|da|>": 50285,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|de|>": 50261,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|el|>": 50281,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|en|>": 50259,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|es|>": 50262,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|et|>": 50307,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|eu|>": 50310,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|fa|>": 50300,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|fi|>": 50277,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|fo|>": 50338,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|fr|>": 50265,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|gl|>": 50319,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|gu|>": 50333,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|haw|>": 50352,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ha|>": 50354,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|he|>": 50279,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|hi|>": 50276,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|hr|>": 50291,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ht|>": 50339,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|hu|>": 50286,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|hy|>": 50312,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|id|>": 50275,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|is|>": 50311,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|it|>": 50274,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ja|>": 50266,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|jw|>": 50356,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ka|>": 50329,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|kk|>": 50316,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|km|>": 50323,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|kn|>": 50306,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ko|>": 50264,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|la|>": 50294,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|lb|>": 50345,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ln|>": 50353,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|lo|>": 50336,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|lt|>": 50293,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|lv|>": 50301,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mg|>": 50349,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mi|>": 50295,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mk|>": 50308,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ml|>": 50296,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mn|>": 50314,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mr|>": 50320,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ms|>": 50282,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mt|>": 50343,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|my|>": 50346,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ne|>": 50313,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|nl|>": 50271,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|nn|>": 50342,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|no|>": 50288,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|oc|>": 50328,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|pa|>": 50321,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|pl|>": 50269,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ps|>": 50340,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|pt|>": 50267,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ro|>": 50284,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ru|>": 50263,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sa|>": 50344,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sd|>": 50332,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|si|>": 50322,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sk|>": 50298,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sl|>": 50305,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sn|>": 50324,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|so|>": 50326,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sq|>": 50317,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sr|>": 50303,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|su|>": 50357,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sv|>": 50273,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sw|>": 50318,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ta|>": 50287,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|te|>": 50299,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|tg|>": 50331,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|th|>": 50289,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|tk|>": 50341,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|tl|>": 50348,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|tr|>": 50268,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|tt|>": 50351,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|uk|>": 50280,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ur|>": 50290,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|uz|>": 50337,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|vi|>": 50278,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|yi|>": 50335,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|yo|>": 50325,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|zh|>": 50260\n', 'ERROR 2024-05-13T19:18:41.238986 },\n', 'ERROR 2024-05-13T19:18:41.238986 "language": "<|en|>",\n', 'ERROR 2024-05-13T19:18:41.238986 "max_initial_timestamp_index": 50,\n', 'ERROR 2024-05-13T19:18:41.238986 "max_length": 448,\n', 'ERROR 2024-05-13T19:18:41.238986 "no_timestamps_token_id": 50363,\n', 'ERROR 2024-05-13T19:18:41.238986 "pad_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:41.238986 "prev_sot_token_id": 50361,\n', 'ERROR 2024-05-13T19:18:41.238986 "return_timestamps": false,\n', 'ERROR 2024-05-13T19:18:41.238986 "suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:41.238986 1,\n', 'ERROR 2024-05-13T19:18:41.238986 2,\n', 'ERROR 2024-05-13T19:18:41.238986 7,\n', 'ERROR 2024-05-13T19:18:41.238986 8,\n', 'ERROR 2024-05-13T19:18:41.238986 9,\n', 'ERROR 2024-05-13T19:18:41.238986 10,\n', 'ERROR 2024-05-13T19:18:41.238986 14,\n', 'ERROR 2024-05-13T19:18:41.238986 25,\n', 'ERROR 2024-05-13T19:18:41.238986 26,\n', 'ERROR 2024-05-13T19:18:41.238986 27,\n', 'ERROR 2024-05-13T19:18:41.238986 28,\n', 'ERROR 2024-05-13T19:18:41.238986 29,\n', 'ERROR 2024-05-13T19:18:41.238986 31,\n', 'ERROR 2024-05-13T19:18:41.238986 58,\n', 'ERROR 2024-05-13T19:18:41.238986 59,\n', 'ERROR 2024-05-13T19:18:41.238986 60,\n', 'ERROR 2024-05-13T19:18:41.238986 61,\n', 'ERROR 2024-05-13T19:18:41.238986 62,\n', 'ERROR 2024-05-13T19:18:41.238986 63,\n', 'ERROR 2024-05-13T19:18:41.238986 90,\n', 'ERROR 2024-05-13T19:18:41.238986 91,\n', 'ERROR 2024-05-13T19:18:41.238986 92,\n', 'ERROR 2024-05-13T19:18:41.238986 93,\n', 'ERROR 2024-05-13T19:18:41.238986 359,\n', 'ERROR 2024-05-13T19:18:41.238986 503,\n', 'ERROR 2024-05-13T19:18:41.238986 522,\n', 'ERROR 2024-05-13T19:18:41.238986 542,\n', 'ERROR 2024-05-13T19:18:41.238986 873,\n', 'ERROR 2024-05-13T19:18:41.238986 893,\n', 'ERROR 2024-05-13T19:18:41.238986 902,\n', 'ERROR 2024-05-13T19:18:41.238986 918,\n', 'ERROR 2024-05-13T19:18:41.238986 922,\n', 'ERROR 2024-05-13T19:18:41.238986 931,\n', 'ERROR 2024-05-13T19:18:41.238986 1350,\n', 'ERROR 2024-05-13T19:18:41.238986 1853,\n', 'ERROR 2024-05-13T19:18:41.238986 1982,\n', 'ERROR 2024-05-13T19:18:41.238986 2460,\n', 'ERROR 2024-05-13T19:18:41.238986 2627,\n', 'ERROR 2024-05-13T19:18:41.238986 3246,\n', 'ERROR 2024-05-13T19:18:41.238986 3253,\n', 'ERROR 2024-05-13T19:18:41.238986 3268,\n', 'ERROR 2024-05-13T19:18:41.238986 3536,\n', 'ERROR 2024-05-13T19:18:41.238986 3846,\n', 'ERROR 2024-05-13T19:18:41.238986 3961,\n', 'ERROR 2024-05-13T19:18:41.238986 4183,\n', 'ERROR 2024-05-13T19:18:41.238986 4667,\n', 'ERROR 2024-05-13T19:18:41.238986 6585,\n', 'ERROR 2024-05-13T19:18:41.238986 6647,\n', 'ERROR 2024-05-13T19:18:41.238986 7273,\n', 'ERROR 2024-05-13T19:18:41.238986 9061,\n', 'ERROR 2024-05-13T19:18:41.238986 9383,\n', 'ERROR 2024-05-13T19:18:41.238986 10428,\n', 'ERROR 2024-05-13T19:18:41.238986 10929,\n', 'ERROR 2024-05-13T19:18:41.238986 11938,\n', 'ERROR 2024-05-13T19:18:41.238986 12033,\n', 'ERROR 2024-05-13T19:18:41.238986 12331,\n', 'ERROR 2024-05-13T19:18:41.238986 12562,\n', 'ERROR 2024-05-13T19:18:41.238986 13793,\n', 'ERROR 2024-05-13T19:18:41.238986 14157,\n', 'ERROR 2024-05-13T19:18:41.238986 14635,\n', 'ERROR 2024-05-13T19:18:41.238986 15265,\n', 'ERROR 2024-05-13T19:18:41.238986 15618,\n', 'ERROR 2024-05-13T19:18:41.238986 16553,\n', 'ERROR 2024-05-13T19:18:41.238986 16604,\n', 'ERROR 2024-05-13T19:18:41.238986 18362,\n', 'ERROR 2024-05-13T19:18:41.238986 18956,\n', 'ERROR 2024-05-13T19:18:41.238986 20075,\n', 'ERROR 2024-05-13T19:18:41.238986 21675,\n', 'ERROR 2024-05-13T19:18:41.238986 22520,\n', 'ERROR 2024-05-13T19:18:41.238986 26130,\n', 'ERROR 2024-05-13T19:18:41.238986 26161,\n', 'ERROR 2024-05-13T19:18:41.238986 26435,\n', 'ERROR 2024-05-13T19:18:41.238986 28279,\n', 'ERROR 2024-05-13T19:18:41.238986 29464,\n', 'ERROR 2024-05-13T19:18:41.238986 31650,\n', 'ERROR 2024-05-13T19:18:41.238986 32302,\n', 'ERROR 2024-05-13T19:18:41.238986 32470,\n', 'ERROR 2024-05-13T19:18:41.238986 36865,\n', 'ERROR 2024-05-13T19:18:41.238986 42863,\n', 'ERROR 2024-05-13T19:18:41.238986 47425,\n', 'ERROR 2024-05-13T19:18:41.238986 49870,\n', 'ERROR 2024-05-13T19:18:41.238986 50254,\n', 'ERROR 2024-05-13T19:18:41.238986 50258,\n', 'ERROR 2024-05-13T19:18:41.238986 50360,\n', 'ERROR 2024-05-13T19:18:41.238986 50361,\n', 'ERROR 2024-05-13T19:18:41.238986 50362\n', 'ERROR 2024-05-13T19:18:41.238986 ],\n', 'ERROR 2024-05-13T19:18:41.238986 "task": "transcribe",\n', 'ERROR 2024-05-13T19:18:41.238986 "task_to_id": {\n', 'ERROR 2024-05-13T19:18:41.238986 "transcribe": 50359,\n', 'ERROR 2024-05-13T19:18:41.238986 "translate": 50358\n', 'ERROR 2024-05-13T19:18:41.238986 },\n', 'ERROR 2024-05-13T19:18:41.238986 "use_scan": false\n', 'ERROR 2024-05-13T19:18:41.238986 }\n', 'ERROR 2024-05-13T19:18:41.238986 loading file vocab.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/vocab.json\n', 'ERROR 2024-05-13T19:18:41.238986 loading file tokenizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer.json\n', 'ERROR 2024-05-13T19:18:41.238986 loading file merges.txt from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/merges.txt\n', 'ERROR 2024-05-13T19:18:41.238986 loading file normalizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/normalizer.json\n', 'ERROR 2024-05-13T19:18:41.238986 loading file added_tokens.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/added_tokens.json\n', 'ERROR 2024-05-13T19:18:41.238986 loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/special_tokens_map.json\n', 'ERROR 2024-05-13T19:18:41.238986 loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer_config.json\n', 'ERROR 2024-05-13T19:18:41.238986 Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n', 'ERROR 2024-05-13T19:18:41.238986 loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/preprocessor_config.json\n', 'ERROR 2024-05-13T19:18:41.238986 Feature extractor WhisperFeatureExtractor {\n', 'ERROR 2024-05-13T19:18:41.238986 "chunk_length": 30,\n', 'ERROR 2024-05-13T19:18:41.238986 "feature_extractor_type": "WhisperFeatureExtractor",\n', 'ERROR 2024-05-13T19:18:41.238986 "feature_size": 80,\n', 'ERROR 2024-05-13T19:18:41.238986 "hop_length": 160,\n', 'ERROR 2024-05-13T19:18:41.238986 "n_fft": 400,\n', 'ERROR 2024-05-13T19:18:41.238986 "n_samples": 480000,\n', 'ERROR 2024-05-13T19:18:41.238986 "nb_max_frames": 3000,\n', 'ERROR 2024-05-13T19:18:41.238986 "padding_side": "right",\n', 'ERROR 2024-05-13T19:18:41.238986 "padding_value": 0.0,\n', 'ERROR 2024-05-13T19:18:41.238986 "processor_class": "WhisperProcessor",\n', 'ERROR 2024-05-13T19:18:41.238986 "return_attention_mask": false,\n', 'ERROR 2024-05-13T19:18:41.238986 "sampling_rate": 16000\n', 'ERROR 2024-05-13T19:18:41.238986 }\n', "ERROR 2024-05-13T19:18:41.238986 Disabling tokenizer parallelism, we're using DataLoader multithreading already\n", 'ERROR 2024-05-13T19:18:43.240189 You have passed task=transcribe, but also have set `forced_decoder_ids` to [[1, 50259], [2, 50359], [3, 50363]] which creates a conflict. `forced_decoder_ids` will be ignored in favor of task=transcribe.\n', 'ERROR 2024-05-13T19:18:47.242157 Train steps ... : 13%|███████▌ | 128/1016 [14:37<4:00:55, 16.28s/it]\n', '2024-05-13T19:18:46.651466 Eval results for step (127 / 1016 | Eval Loss: 4.316526412963867 | Eval clap: 0.4042370617389679 | Eval wer: 28.030303030303028 |)\n']}}, 'dropped': 0}} +2024-05-13 21:19:50,392 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:50,427 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:52,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:19:52,116 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:19:52,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:52,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:54,697 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:54,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:55,034 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:55,276 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:56,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:56,871 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:58,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:58,991 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:00,703 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:01,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:01,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:01,278 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:03,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:03,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:05,280 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:05,857 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:06,847 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:20:07,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:20:07,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:20:07,281 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:07,283 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:07,319 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:09,088 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:20:09,089 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:20:09,089 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:20:09,089 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:20:09,282 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:20:09,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:09,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:11,090 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:11,283 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:11,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:11,482 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:13,614 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:13,660 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:15,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:15,808 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:16,091 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:17,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:17,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:19,639 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:20:20,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:20,046 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:21,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:21,289 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:22,071 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:22,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:22,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:20:22,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:20:24,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:24,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:26,271 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:26,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:26,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:28,506 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:28,535 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:30,607 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:30,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:31,294 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:31,752 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:32,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:32,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:34,755 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:34,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:36,811 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:36,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:36,889 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:37,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:20:37,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:20:37,297 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:40,910 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:20:40,932 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:40,943 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:41,299 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:42,189 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:42,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:42,982 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:43,431 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:20:43,432 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:20:43,432 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:20:43,433 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:20:44,301 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:20:45,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:45,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:45,301 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:47,126 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:47,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:47,302 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:47,800 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:49,215 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:49,265 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:49,642 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:20:51,364 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:51,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:52,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:20:52,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:20:53,270 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:53,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:53,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:55,306 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:55,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:55,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:57,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:57,809 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:58,296 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:59,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:59,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:02,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:02,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:03,298 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:04,168 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:04,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:06,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:06,275 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:07,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:21:07,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:21:07,312 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:08,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:08,325 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:08,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:10,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:10,404 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:14,163 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:14,549 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:21:14,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:14,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:15,316 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:16,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:16,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:18,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:18,697 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:18,923 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:21:18,923 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:21:18,924 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:21:18,924 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:21:19,318 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:19,318 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:21:19,645 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:21:19,646 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:21,319 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:22,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:22,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:22,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:21:22,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:21:23,320 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:24,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:24,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:25,321 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:25,598 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:26,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:26,298 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:27,322 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:28,365 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:28,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:30,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:30,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:31,453 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:32,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:32,680 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:34,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:34,810 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:36,454 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:36,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:36,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:37,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:21:37,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:21:38,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:39,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:41,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:41,099 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:42,341 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:43,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:43,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:45,331 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:45,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:45,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:47,564 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:48,106 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:21:48,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:48,148 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:49,647 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:21:50,170 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:50,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:52,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:21:52,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:21:52,214 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:52,226 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:52,254 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:21:52,323 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:21:52,323 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:21:52,324 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:21:52,334 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:21:53,325 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:53,335 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:54,248 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:54,271 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:56,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:56,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:58,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:58,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:58,389 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:59,337 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:00,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:00,386 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:02,443 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:02,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:03,339 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:04,030 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:04,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:04,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:05,340 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:06,593 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:06,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:07,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:22:07,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:22:07,342 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:08,638 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:08,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:09,966 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:10,681 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:10,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:12,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:12,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:14,750 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:14,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:14,967 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:16,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:16,800 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:19,649 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:22:20,650 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:20,912 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:22:20,965 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:20,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:22,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:22:22,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:22:23,019 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:23,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:23,828 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:22:23,831 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:22:23,831 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:22:23,831 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:22:24,351 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:22:25,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:25,102 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:25,351 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:25,832 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:27,175 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:27,209 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:29,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:29,317 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:30,834 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:31,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:31,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:33,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:33,571 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:35,356 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:35,677 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:35,711 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:36,030 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:37,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:22:37,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:22:37,746 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:37,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:39,358 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:39,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:39,938 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:41,730 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:42,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:42,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:43,360 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:44,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:44,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:46,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:46,291 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:46,796 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:47,362 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:48,054 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:22:48,055 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:22:48,055 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:22:48,056 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:22:48,363 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:22:48,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:48,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:49,363 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:49,652 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:22:50,524 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:50,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:52,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:22:52,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:22:52,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:54,562 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:22:54,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:54,671 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:56,729 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:56,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:57,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:58,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:58,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:00,839 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:00,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:01,370 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:02,880 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:02,905 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:03,249 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:04,935 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:04,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:06,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:06,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:07,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:23:07,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:23:08,288 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:09,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:09,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:11,375 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:12,388 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:12,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:13,488 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:14,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:14,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:16,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:16,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:17,378 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:18,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:18,516 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:18,877 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:19,654 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:23:20,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:20,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:22,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:23:22,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:23:22,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:22,589 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:23,381 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:24,250 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:24,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:23:24,681 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:23:24,681 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:23:24,682 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:23:25,382 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:25,382 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:23:25,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:25,839 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:27,383 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:29,384 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:29,666 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:29,684 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:23:29,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:29,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:31,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:31,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:33,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:33,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:34,667 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:35,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:35,897 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:37,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:23:37,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:23:37,926 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:37,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:39,388 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:39,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:39,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:39,985 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:42,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:42,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:44,066 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:44,089 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:44,986 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:46,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:46,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:48,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:48,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:49,656 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:23:50,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:50,305 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:50,658 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:51,393 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:52,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:23:52,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:23:52,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:52,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:54,418 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:54,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:56,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:56,458 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:56,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:58,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:58,512 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:59,397 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:01,976 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:02,382 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:24:02,442 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:02,463 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:02,949 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:24:02,950 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:24:02,950 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:24:02,951 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:24:03,399 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:03,399 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:24:05,400 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:05,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:05,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:07,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:24:07,119 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:24:07,248 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:07,401 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:07,750 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:07,764 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:09,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:09,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:11,402 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:11,819 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:11,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:13,058 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:15,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:15,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:17,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:17,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:18,059 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:19,346 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:19,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:19,659 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:24:21,396 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:21,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:22,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:24:22,119 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:24:23,258 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:23,435 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:23,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:25,482 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:25,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:28,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:28,819 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:28,832 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:30,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:30,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:31,412 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:32,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:32,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:34,038 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:34,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:34,945 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:37,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:24:37,119 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:24:38,254 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:24:38,256 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:24:38,256 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:24:38,256 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:24:38,416 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:24:38,910 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:24:38,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:38,948 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:39,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:39,417 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:40,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:40,984 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:43,008 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:43,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:44,577 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:45,051 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:45,419 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:46,276 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:48,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:48,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:49,421 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:49,661 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:24:49,662 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:50,366 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:50,386 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:51,422 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:52,119 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:24:52,119 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:24:52,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:52,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:53,423 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:54,458 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:55,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:55,862 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:58,019 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:58,037 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:00,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:00,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:00,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:02,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:02,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:04,433 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:04,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:05,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:06,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:06,599 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:07,119 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:25:07,119 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:25:07,670 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:25:07,671 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:25:07,671 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:25:07,672 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:25:08,432 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:25:08,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:08,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:09,433 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:10,672 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:10,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:10,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:14,729 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:25:15,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:15,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:15,673 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:17,269 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:17,292 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:17,437 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:19,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:19,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:19,664 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:25:21,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:21,589 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:21,666 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:22,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:25:22,120 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:25:23,440 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:23,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:23,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:25,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:25,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:27,270 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:27,442 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:28,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:28,066 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:30,153 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:30,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:31,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:31,522 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:25:31,523 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:25:31,523 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:25:31,524 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:25:32,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:32,271 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:32,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:25:32,524 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:33,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:34,327 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:34,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:36,508 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:36,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:37,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:25:37,120 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:25:38,283 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:38,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:38,702 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:40,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:40,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:42,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:42,952 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:43,284 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:45,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:45,032 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:47,455 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:48,685 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:48,743 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:25:49,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:49,234 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:49,667 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:25:51,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:51,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:52,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:25:52,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:25:53,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:53,479 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:54,334 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:55,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:55,568 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:57,461 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:57,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:57,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:59,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:59,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:00,033 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:01,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:01,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:03,465 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:04,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:04,047 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:05,725 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:06,161 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:06,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:07,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:26:07,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:26:08,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:08,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:09,468 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:10,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:10,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:10,895 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:10,953 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:26:10,954 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:26:10,955 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:26:10,955 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:26:11,469 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:11,469 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:26:12,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:12,613 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:13,470 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:14,670 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:14,696 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:15,472 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:16,415 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:16,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:16,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:18,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:18,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:19,669 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:26:21,670 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:22,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:26:22,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:26:22,857 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:26:22,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:22,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:24,917 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:24,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:25,477 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:26,976 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:26,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:27,638 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:29,045 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:29,075 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:31,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:31,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:32,639 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:33,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:33,343 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:35,457 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:35,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:37,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:26:37,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:26:37,484 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:37,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:37,681 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:38,340 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:39,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:39,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:41,926 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:41,949 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:43,487 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:43,751 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:44,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:44,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:46,100 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:46,134 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:47,626 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:26:47,627 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:26:47,627 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:26:47,627 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:26:48,211 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:48,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:48,489 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:26:49,490 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:49,630 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:49,672 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:26:50,312 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:50,330 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:51,491 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:52,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:26:52,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:26:52,353 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:52,370 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:55,461 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:55,493 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:56,357 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:26:56,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:56,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:57,494 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:58,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:58,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:00,462 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:00,642 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:00,665 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:02,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:02,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:04,872 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:04,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:05,463 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:06,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:07,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:07,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:27:07,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:27:09,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:09,144 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:11,298 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:11,300 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:11,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:13,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:13,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:15,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:15,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:16,299 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:17,660 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:17,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:19,504 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:19,674 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:27:19,774 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:19,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:21,676 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:21,910 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:21,927 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:22,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:27:22,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:27:24,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:24,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:26,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:26,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:26,719 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:26,815 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:27:26,816 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:27:26,816 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:27:26,816 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:27:27,508 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:27,508 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:27:29,509 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:30,235 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:27:30,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:30,316 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:31,817 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:32,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:32,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:34,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:34,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:35,515 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:36,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:36,466 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:36,881 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:37,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:27:37,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:27:37,517 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:38,507 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:38,522 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:40,653 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:40,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:41,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:42,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:42,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:42,876 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:43,524 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:44,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:44,812 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:46,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:46,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:47,877 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:49,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:49,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:49,678 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:27:51,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:51,157 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:52,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:27:52,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:27:53,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:53,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:53,866 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:55,275 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:55,287 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:57,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:57,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:58,866 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:59,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:59,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:01,562 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:28:01,563 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:28:01,563 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:28:01,564 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:28:02,537 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:28:03,538 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:04,565 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:04,688 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:28:04,709 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:04,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:06,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:06,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:07,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:28:07,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:28:08,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:08,806 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:10,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:10,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:10,845 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:11,541 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:12,875 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:12,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:14,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:14,937 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:16,011 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:16,965 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:16,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:17,544 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:19,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:19,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:19,679 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:28:21,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:21,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:21,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:21,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:22,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:28:22,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:28:24,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:24,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:25,547 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:26,219 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:26,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:26,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:26,517 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:28:26,518 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:28:26,518 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:28:26,518 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:28:26,548 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:28:27,548 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:28,456 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:28,470 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:29,549 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:30,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:30,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:31,519 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:32,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:32,547 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:34,576 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:34,590 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:36,520 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:37,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:28:37,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:28:38,605 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:28:38,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:38,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:40,673 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:40,684 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:41,559 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:42,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:42,729 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:43,556 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:44,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:44,764 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:46,561 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:47,068 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:48,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:49,682 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:28:50,282 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:50,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:51,683 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:52,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:28:52,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:28:52,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:52,329 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:53,561 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:54,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:54,365 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:56,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:56,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:56,936 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:58,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:58,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:00,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:00,507 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:01,565 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:02,522 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:02,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:02,545 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:04,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:04,588 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:05,567 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:06,618 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:06,638 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:07,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:29:07,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:29:07,554 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:07,612 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:29:07,613 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:29:07,613 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:29:07,613 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:29:08,568 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:29:08,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:08,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:09,569 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:11,570 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:12,789 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:29:12,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:12,829 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:13,021 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:14,881 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:14,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:16,926 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:16,943 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:18,022 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:18,970 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:18,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:19,685 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:29:21,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:21,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:21,575 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:22,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:29:22,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:29:23,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:23,081 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:23,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:25,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:25,176 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:27,206 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:27,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:28,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:29,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:29,257 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:31,292 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:31,305 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:33,404 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:33,434 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:33,581 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:33,653 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:35,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:35,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:37,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:29:37,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:29:37,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:37,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:39,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:39,833 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:39,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:41,585 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:41,984 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:42,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:44,263 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:44,264 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:29:44,265 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:29:44,266 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:29:44,266 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:29:44,587 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:29:45,587 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:46,216 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:29:46,293 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:46,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:48,409 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:48,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:49,589 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:49,688 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:49,689 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:29:50,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:50,537 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:51,590 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:52,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:29:52,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:29:52,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:52,660 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:53,591 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:54,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:54,730 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:54,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:56,824 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:56,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:58,935 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:58,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:59,731 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:01,028 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:01,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:03,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:03,158 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:04,732 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:05,233 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:05,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:07,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:30:07,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:30:07,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:07,492 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:09,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:09,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:10,270 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:11,599 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:11,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:11,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:13,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:13,964 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:16,016 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:16,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:16,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:18,909 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:30:18,910 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:30:18,911 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:30:18,911 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:30:19,602 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:19,603 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:30:19,690 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:30:19,842 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:30:20,312 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:20,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:21,603 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:21,692 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:22,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:30:22,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:30:22,416 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:22,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:24,563 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:24,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:25,605 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:26,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:26,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:27,252 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:28,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:28,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:29,606 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:30,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:30,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:32,921 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:32,937 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:33,218 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:33,608 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:35,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:35,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:35,610 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:37,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:30:37,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:30:37,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:37,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:38,313 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:39,208 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:39,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:41,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:41,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:43,314 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:43,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:43,464 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:45,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:45,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:47,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:47,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:48,315 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:49,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:30:49,171 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:30:49,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:30:49,172 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:30:49,617 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:49,617 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:30:49,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:49,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:49,692 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:30:51,618 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:52,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:30:52,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:30:53,486 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:30:53,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:54,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:54,287 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:56,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:56,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:58,233 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:58,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:59,622 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:00,096 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:00,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:00,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:02,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:02,425 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:04,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:04,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:05,624 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:05,814 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:06,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:06,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:07,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:31:07,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:31:08,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:08,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:09,626 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:10,765 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:10,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:11,627 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:11,796 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:12,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:12,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:13,094 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:31:13,096 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:31:13,096 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:31:13,097 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:31:13,628 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:13,629 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:31:14,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:15,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:15,630 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:17,098 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:17,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:17,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:19,242 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:19,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:19,694 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:31:21,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:21,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:22,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:31:22,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:31:22,332 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:23,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:23,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:27,213 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:31:27,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:27,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:27,636 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:27,952 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:30,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:30,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:32,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:32,253 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:32,953 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:34,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:34,290 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:36,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:36,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:37,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:31:37,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:31:38,157 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:38,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:38,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:39,641 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:40,394 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:40,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:42,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:42,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:43,158 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:44,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:44,547 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:45,644 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:46,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:46,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:48,597 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:48,855 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:48,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:49,696 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:31:50,982 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:51,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:51,647 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:52,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:31:52,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:31:52,736 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:31:52,739 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:31:52,739 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:31:52,739 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:31:53,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:53,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:53,648 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:53,648 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:31:53,740 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:55,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:55,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:55,649 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:57,269 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:57,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:59,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:01,416 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:32:01,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:01,495 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:03,517 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:03,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:04,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:05,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:05,568 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:07,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:32:07,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:32:07,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:07,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:07,656 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:09,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:09,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:09,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:11,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:11,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:13,890 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:13,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:14,347 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:15,997 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:16,034 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:18,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:18,187 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:19,662 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:19,699 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:32:19,700 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:20,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:20,276 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:22,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:32:22,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:32:22,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:22,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:24,458 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:24,496 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:25,271 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:26,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:26,696 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:27,665 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:28,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:28,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:30,729 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:31,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:31,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:32,204 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:32:32,205 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:32:32,205 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:32:32,205 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:32:32,668 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:32:33,669 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:35,325 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:32:35,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:35,382 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:35,935 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:37,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:32:37,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:32:37,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:37,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:37,671 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:39,463 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:39,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:39,672 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:41,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:41,522 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:41,542 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:43,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:43,617 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:43,674 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:45,741 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:45,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:46,620 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:47,823 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:47,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:49,700 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:32:49,945 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:49,972 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:51,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:52,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:52,049 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:52,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:32:52,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:32:54,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:54,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:56,204 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:56,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:57,319 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:58,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:58,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:00,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:00,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:02,320 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:02,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:02,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:04,443 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:04,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:05,683 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:07,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:33:07,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:33:08,317 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:08,488 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:33:08,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:08,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:11,782 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:11,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:13,688 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:13,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:13,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:14,274 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:15,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:15,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:17,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:17,991 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:19,275 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:19,703 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:33:20,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:20,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:22,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:22,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:22,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:33:22,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:33:24,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:24,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:24,292 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:26,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:26,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:28,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:28,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:29,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:30,224 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:30,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:33,613 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:33,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:34,294 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:35,661 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:35,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:37,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:33:37,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:33:37,698 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:37,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:37,739 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:39,699 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:39,758 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:39,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:40,126 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:41,700 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:42,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 21:33:42,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 21:33:42,193 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:33:42,241 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:33:42,242 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:33:42,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav +2024-05-13 21:33:42,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav +2024-05-13 21:33:42,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 21:33:43,175 INFO wandb-upload_1:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmpw31229o0 +2024-05-13 21:33:43,351 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/izo3er6j-media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json +2024-05-13 21:33:43,759 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep254 - {'id': 'QXJ0aWZhY3Q6ODM0Nzg0OTMy', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI1MzkxMg==', 'latestArtifact': None}} +2024-05-13 21:33:43,760 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:43,760 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json with policy now +2024-05-13 21:33:43,760 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:33:43,760 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:33:43,813 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/jd743gyg-media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav +2024-05-13 21:33:43,857 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/5u23cw49-media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav +2024-05-13 21:33:43,939 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/rp7nv9pt-media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav +2024-05-13 21:33:43,940 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/u1p9sxmq-media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav +2024-05-13 21:33:43,973 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/nqi7q8oq-media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav +2024-05-13 21:33:43,977 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/79cry1c2-media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav +2024-05-13 21:33:44,006 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/t9xp6l6n-media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav +2024-05-13 21:33:44,197 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8k8rzequ-media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json +2024-05-13 21:33:44,707 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:33:44,753 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:33:44,753 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,753 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav with policy now +2024-05-13 21:33:44,753 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,753 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav with policy now +2024-05-13 21:33:44,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,754 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav with policy now +2024-05-13 21:33:44,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,754 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav with policy now +2024-05-13 21:33:44,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,754 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav with policy now +2024-05-13 21:33:44,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,754 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav with policy now +2024-05-13 21:33:44,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,755 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav with policy now +2024-05-13 21:33:44,865 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:33:44,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:44,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:45,708 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:45,755 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:45,778 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ikvtwf18-media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav +2024-05-13 21:33:45,784 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/niixjtsd-media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav +2024-05-13 21:33:45,785 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/exh9xy90-media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav +2024-05-13 21:33:45,812 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/rwtt25mx-media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav +2024-05-13 21:33:45,826 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/koxx4in5-media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav +2024-05-13 21:33:45,831 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/nl2ym3yq-media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav +2024-05-13 21:33:45,840 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/0owfk6bw-media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav +2024-05-13 21:33:46,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:46,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:47,575 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:33:47,577 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:33:47,578 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:33:47,578 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:33:47,709 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:47,709 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:33:49,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:49,044 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:49,707 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:33:49,712 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:51,067 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:51,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:51,569 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:51,713 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:52,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:33:52,126 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:33:54,484 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:54,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:56,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:56,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:57,265 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:58,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:58,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:00,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:00,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:02,266 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:02,710 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:02,728 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:04,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:04,762 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:06,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:06,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:07,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:34:07,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:34:07,297 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:08,829 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:08,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:10,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:10,908 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:12,297 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:12,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:12,991 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:13,728 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:16,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:16,335 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:17,414 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:19,708 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:34:20,269 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:34:20,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:20,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:21,734 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:22,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:34:22,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:34:22,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:22,369 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:23,295 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:24,391 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:24,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:26,425 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:26,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:27,738 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:28,460 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:28,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:28,684 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:29,627 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:34:29,627 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:34:29,628 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:34:29,628 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:34:29,739 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:34:31,741 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:31,870 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:31,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:33,742 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:33,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:33,959 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:34,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:35,744 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:36,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:36,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:37,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:34:37,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:34:38,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:38,093 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:40,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:40,146 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:40,357 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:42,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:42,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:44,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:44,233 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:45,357 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:46,257 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:46,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:48,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:48,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:49,712 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:34:50,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:50,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:50,713 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:52,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:34:52,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:34:53,757 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:54,475 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:34:54,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:54,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:56,292 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:56,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:56,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:58,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:58,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:00,924 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:00,951 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:01,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:02,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:03,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:03,764 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:05,049 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:05,064 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:06,721 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:07,094 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:07,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:07,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:35:07,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:35:08,163 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:35:08,164 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:35:08,164 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:35:08,165 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:35:08,768 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:35:09,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:09,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:09,769 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:11,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:11,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:12,165 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:13,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:13,257 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:13,772 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:15,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:15,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:17,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:17,366 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:17,537 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:17,774 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:19,432 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:19,469 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:19,714 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:35:19,776 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:21,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:21,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:22,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:35:22,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:35:23,256 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:23,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:23,673 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:27,909 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:35:27,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:27,995 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:28,258 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:30,069 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:30,084 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:32,218 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:32,365 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:33,457 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:33,786 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:34,429 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:34,442 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:36,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:36,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:37,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:35:37,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:35:38,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:38,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:39,287 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:40,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:40,633 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:42,666 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:42,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:43,657 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:35:43,658 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:35:43,658 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:35:43,658 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:35:43,792 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:35:44,659 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:44,705 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:44,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:45,793 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:46,737 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:46,753 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:48,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:48,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:49,660 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:49,716 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:35:50,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:50,836 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:51,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:52,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:35:52,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:35:52,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:52,870 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:54,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:54,935 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:55,534 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:55,800 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:56,995 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:57,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:59,803 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:00,762 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:36:01,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:01,216 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:01,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:01,805 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:03,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:03,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:05,424 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:05,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:06,178 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:07,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:36:07,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:36:07,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:07,508 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:09,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:09,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:09,905 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:36:09,906 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:36:09,906 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:36:09,906 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:36:10,811 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:36:11,776 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:11,811 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:11,815 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:11,907 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:13,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:13,944 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:16,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:16,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:16,909 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:18,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:18,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:19,719 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:36:20,300 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:20,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:21,918 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:22,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:36:22,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:36:22,464 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:22,491 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:23,818 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:24,588 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:24,634 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:26,718 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:26,758 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:27,308 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:28,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:28,890 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:29,825 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:30,975 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:31,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:32,738 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:33,827 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:34,616 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:36:34,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:34,690 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:36,738 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:36,758 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:37,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:36:37,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:36:37,830 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:38,267 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:38,819 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:38,830 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:39,408 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:36:39,409 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:36:39,409 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:36:39,410 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:36:39,831 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:39,831 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:36:40,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:40,879 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:41,832 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:42,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:42,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:44,241 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:44,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:45,000 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:47,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:47,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:49,069 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:49,081 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:49,242 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:49,722 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:36:51,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:51,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:52,126 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:36:52,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:36:53,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:53,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:54,602 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:55,188 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:55,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:57,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:57,279 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:59,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:59,440 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:00,448 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:01,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:01,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:01,844 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:03,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:03,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:05,449 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:07,126 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:37:07,126 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:37:07,900 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:37:07,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:07,968 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:09,849 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:10,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:10,072 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:10,513 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:12,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:12,150 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:13,851 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:14,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:14,259 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:16,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:16,373 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:16,397 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:17,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:37:17,258 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:37:17,259 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:37:17,259 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:37:17,854 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:17,854 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:37:18,545 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:18,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:19,726 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:37:19,855 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:20,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:20,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:21,856 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:22,059 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:22,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:37:22,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:37:22,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:22,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:23,858 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:24,957 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:24,984 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:27,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:27,131 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:27,306 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:29,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:29,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:31,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:31,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:32,306 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:33,460 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:33,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:35,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:35,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:37,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:37:37,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:37:37,310 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:37,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:37,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:39,867 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:41,861 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:37:41,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:41,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:42,499 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:44,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:44,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:46,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:46,157 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:47,521 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:47,871 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:48,282 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:48,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:49,726 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:37:50,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:50,476 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:52,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:37:52,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:37:52,303 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:37:52,304 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:37:52,304 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:37:52,304 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:37:52,571 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:52,606 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:52,874 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:52,874 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:37:53,305 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:53,875 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:54,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:54,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:56,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:56,819 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:57,877 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:58,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:58,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:58,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:00,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:00,946 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:01,879 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:02,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:02,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:03,881 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:03,906 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:06,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:06,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:07,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:38:07,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:38:08,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:08,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:09,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:10,356 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:10,377 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:12,402 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:12,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:14,349 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:16,370 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:38:16,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:16,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:17,889 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:18,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:18,550 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:19,729 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:38:19,731 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:20,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:20,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:22,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:38:22,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:38:22,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:22,614 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:24,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:24,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:25,287 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:26,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:26,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:27,824 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:38:27,825 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:38:27,825 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:38:27,825 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:38:27,895 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:38:28,896 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:29,897 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:30,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:30,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:30,826 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:32,153 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:32,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:34,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:34,204 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:35,900 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:36,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:36,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:36,422 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:37,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:38:37,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:38:38,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:38,279 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:40,302 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:40,312 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:41,475 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:41,903 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:42,335 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:42,346 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:43,904 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:45,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:45,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:46,804 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:47,700 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:47,710 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:47,907 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:49,731 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:38:51,677 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:38:51,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:51,718 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:52,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:38:52,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:52,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:38:53,740 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:53,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:55,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:55,808 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:56,617 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:38:56,618 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:38:56,618 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:38:56,618 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:38:56,912 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:56,912 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:38:57,619 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:57,829 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:57,846 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:57,913 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:59,875 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:59,886 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:01,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:01,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:02,619 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:03,968 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:03,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:06,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:06,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:07,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:39:07,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:39:08,171 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:08,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:08,299 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:10,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:10,322 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:11,920 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:12,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:12,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:13,500 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:14,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:14,444 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:16,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:16,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:18,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:18,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:19,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:19,734 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:39:19,924 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:20,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:20,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:22,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:39:22,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:39:24,533 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:25,928 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:25,999 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:39:26,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:26,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:27,930 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:28,069 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:28,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:30,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:30,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:30,178 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:30,242 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:39:30,243 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:39:30,243 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:39:30,244 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:39:30,932 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:30,932 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:39:31,932 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:32,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:32,274 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:33,933 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:34,533 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:34,574 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:35,341 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:37,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:39:37,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:39:37,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:37,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:39,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:39,733 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:41,265 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:41,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:41,807 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:43,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:43,842 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:45,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:45,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:46,268 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:47,965 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:48,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:49,737 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:39:50,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:50,044 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:51,739 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:52,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:52,089 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:52,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:39:52,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:39:53,946 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:54,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:54,187 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:56,215 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:56,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:56,859 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:00,414 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:40:00,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:00,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:01,950 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:02,481 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:02,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:02,785 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:04,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:04,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:06,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:06,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:07,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:40:07,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:40:07,954 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:08,312 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:08,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:08,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:10,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:10,728 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:11,212 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:40:11,213 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:40:11,213 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:40:11,214 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:40:11,957 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:11,957 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:40:12,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:12,781 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:12,958 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:13,944 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:14,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:14,812 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:15,960 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:16,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:16,859 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:17,961 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:18,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:18,925 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:19,067 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:19,739 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:40:20,955 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:20,966 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:22,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:40:22,129 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:40:23,066 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:23,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:24,301 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:25,131 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:25,156 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:27,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:27,216 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:29,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:29,269 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:29,302 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:33,608 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:40:33,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:33,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:33,971 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:34,516 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:35,722 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:35,747 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:37,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:40:37,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:40:38,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:38,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:40,296 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:41,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:41,159 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:41,976 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:43,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:43,287 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:45,373 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:45,442 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:45,838 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:46,890 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:40:46,891 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:40:46,891 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:40:46,892 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:40:46,979 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:40:47,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:47,527 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:47,980 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:48,981 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:49,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:49,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:49,742 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:40:51,395 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:51,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:51,914 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:51,983 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:52,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:40:52,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:40:54,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:54,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:55,985 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:56,215 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:56,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:56,458 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:57,986 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:58,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:58,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:00,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:00,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:01,614 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:02,392 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:02,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:04,435 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:04,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:06,615 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:07,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:41:07,129 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:41:08,553 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:41:08,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:08,588 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:09,994 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:10,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:10,632 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:12,357 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:12,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:12,670 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:14,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:14,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:16,742 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:16,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:17,358 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:18,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:18,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:19,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:41:19,106 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:41:19,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:41:19,106 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:41:19,743 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:41:20,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:20,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:41:20,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:20,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:21,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:22,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:41:22,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:41:23,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:23,099 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:23,277 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:25,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:25,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:26,006 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:27,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:27,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:28,977 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:29,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:29,432 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:30,008 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:31,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:31,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:34,011 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:34,514 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:34,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:34,592 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:36,012 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:37,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:41:37,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:41:37,503 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:37,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:40,309 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:40,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:40,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:44,977 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:41:44,978 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:41:44,978 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:41:44,978 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:41:45,018 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:41:45,979 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:46,019 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:47,019 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:47,633 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:41:47,679 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:47,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:49,746 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:41:50,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:50,872 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:51,748 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:52,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:41:52,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:41:53,808 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:53,838 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:56,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:56,816 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:57,321 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:59,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:59,871 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:00,027 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:02,566 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:02,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:02,827 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:05,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:05,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:06,030 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:07,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:42:07,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:42:08,253 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:08,838 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:08,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:11,942 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:11,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:12,034 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:14,035 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:14,074 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:14,994 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:15,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:15,715 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:42:15,716 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:42:15,716 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:42:15,717 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:42:16,036 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:42:17,037 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:17,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:18,012 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:18,038 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:19,597 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:19,748 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:42:20,039 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:21,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:21,066 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:22,130 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:42:22,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:42:24,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:24,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:25,299 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:27,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:27,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:30,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:30,216 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:30,300 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:35,850 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:36,049 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:37,130 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:42:37,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:42:39,413 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:42:40,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:40,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:41,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:43,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:44,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:44,054 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:46,493 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:47,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:47,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:49,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:42:50,058 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:50,074 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:50,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:51,736 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:51,781 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:42:51,782 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:42:51,782 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:42:51,782 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:42:52,060 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:42:52,130 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:42:52,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:42:53,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:53,060 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:53,071 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:54,061 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:56,062 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:56,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:56,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:57,592 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:58,063 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:59,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:59,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:02,131 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:02,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:02,593 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:05,166 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:05,208 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:07,130 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:43:07,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:43:08,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:08,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:08,294 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:11,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:11,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:14,073 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:14,159 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:14,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:14,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:17,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:17,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:19,160 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:19,758 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:43:20,496 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:20,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:22,078 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:22,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:43:22,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:43:23,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:23,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:24,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:26,802 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:43:26,803 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:43:26,803 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:43:26,803 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:43:27,081 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:43:28,082 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:29,082 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:29,804 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:32,084 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:33,077 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:43:33,856 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:33,874 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:35,530 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:36,086 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:36,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:36,806 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:37,130 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:43:37,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:43:38,088 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:39,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:39,926 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:41,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:42,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:42,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:45,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:46,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:46,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:49,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:49,100 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:49,759 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:43:50,095 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:51,761 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:52,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:43:52,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:43:52,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:52,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:55,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:55,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:57,286 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:58,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:58,287 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:59,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:43:59,148 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:43:59,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:43:59,148 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:44:00,101 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:00,101 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:44:01,102 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:01,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:01,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:03,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:04,276 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:04,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:06,105 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:07,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:44:07,132 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:44:07,251 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:07,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:08,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:10,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:10,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:12,108 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:13,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:13,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:13,868 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:14,109 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:16,110 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:16,209 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:16,260 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:19,254 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:19,763 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:44:22,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:44:22,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:44:24,262 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:24,968 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:44:24,969 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:44:24,969 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:44:24,970 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:44:25,104 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:44:25,117 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:44:25,138 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:25,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:26,118 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:27,118 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:28,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:28,153 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:29,970 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:31,176 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:31,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:34,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:34,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:34,972 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:37,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:37,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:44:37,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:44:37,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:38,125 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:39,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:39,864 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:40,341 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:42,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:42,542 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:45,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:45,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:45,550 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:46,130 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:47,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:48,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:49,764 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:44:50,765 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:50,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:50,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:52,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:44:52,132 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:44:52,134 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:53,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:53,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:56,136 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:56,197 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:56,249 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:44:56,250 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:44:56,250 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:44:56,251 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:44:56,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:56,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:57,136 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:57,137 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:44:58,137 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:59,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:59,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:00,138 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:01,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:01,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:02,165 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:04,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:04,700 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:07,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:45:07,132 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:45:07,284 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:12,284 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:13,098 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:45:13,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:13,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:15,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:15,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:17,635 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:18,148 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:18,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:18,827 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:19,767 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:45:21,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:21,683 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:22,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:45:22,132 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:45:23,355 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:24,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:24,542 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:26,153 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:27,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:27,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:28,614 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:30,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:30,034 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:30,155 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:32,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:32,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:32,973 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:45:32,974 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:45:32,975 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:45:32,975 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:45:33,156 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:45:33,975 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:34,157 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:35,145 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:35,157 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:35,166 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:37,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:45:37,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:45:37,812 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:37,834 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:38,159 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:39,705 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:40,160 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:40,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:40,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:43,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:43,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:44,706 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:45,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:45,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:48,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:48,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:49,707 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:49,769 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:45:51,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:51,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:52,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:45:52,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:45:54,168 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:55,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:59,478 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:45:59,503 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:59,514 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:00,260 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:02,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:02,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:04,174 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:04,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:04,694 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:05,602 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:07,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:46:07,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:46:07,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:46:07,226 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:07,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:07,333 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:46:07,334 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:46:07,334 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:46:08,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:08,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:46:09,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:09,816 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:09,838 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:11,046 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:12,179 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:12,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:12,535 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:15,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:15,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:16,181 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:16,934 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:17,756 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:17,771 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:18,183 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:19,771 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:46:20,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:20,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:22,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:46:22,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:22,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:46:23,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:23,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:25,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:25,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:27,318 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:28,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:28,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:31,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:31,190 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:32,192 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:32,704 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:33,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:33,848 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:36,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:36,420 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:37,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:46:37,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:46:38,311 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:39,690 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:46:39,691 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:46:39,692 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:46:39,692 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:46:40,196 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:46:41,196 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:42,197 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:43,693 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:45,517 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:46:45,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:45,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:48,200 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:48,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:48,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:49,470 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:49,774 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:46:50,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:50,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:52,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:46:52,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:46:52,202 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:53,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:53,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:54,496 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:56,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:56,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:56,204 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:58,206 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:58,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:58,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:59,942 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:01,444 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:01,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:04,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:04,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:04,944 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:06,427 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:47:06,429 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:47:06,429 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:47:06,429 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:47:06,925 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:06,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:07,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:47:07,134 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:47:07,210 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:07,210 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:47:08,211 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:09,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:09,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:10,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:12,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:12,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:15,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:15,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:15,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:17,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:17,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:19,777 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:47:20,315 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:20,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:20,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:22,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:47:22,134 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:47:22,219 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:23,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:23,290 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:25,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:30,223 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:30,856 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:31,276 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:47:32,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:32,550 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:34,225 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:35,084 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:35,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:36,013 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:37,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:47:37,134 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:47:37,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:37,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:38,227 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:39,315 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:47:39,315 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:47:39,316 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:47:39,316 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:47:40,229 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:40,229 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:47:40,386 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:40,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:41,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:41,229 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:42,230 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:42,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:42,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:45,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:45,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:46,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:47,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:47,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:49,779 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:47:50,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:50,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:50,234 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:51,780 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:52,134 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:47:52,135 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:47:52,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:52,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:55,201 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:55,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:56,942 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:57,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:57,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:00,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:00,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:01,944 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:02,458 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:02,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:04,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:04,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:06,944 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:07,134 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:48:07,134 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:48:07,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:07,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:12,321 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:14,778 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:48:15,730 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:15,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:17,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:18,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:18,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:19,781 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:48:20,633 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:20,658 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:22,134 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:48:22,135 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:48:22,571 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:23,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:23,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:24,252 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:25,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:25,343 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:26,253 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:27,765 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:27,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:28,254 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:28,525 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:28,647 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 21:48:28,648 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 21:48:28,654 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:48:28,722 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:48:28,724 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:48:29,255 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_492597073098578f0605.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json +2024-05-13 21:48:29,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 21:48:29,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 21:48:29,939 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/d4nh4djq-media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json +2024-05-13 21:48:30,170 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:30,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:30,345 INFO wandb-upload_6:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmpwdcyog3o +2024-05-13 21:48:30,503 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/9vkesifq-media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav +2024-05-13 21:48:30,545 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/jp8mgrso-media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav +2024-05-13 21:48:30,564 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/mr2sxip5-media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav +2024-05-13 21:48:30,564 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/xqs2j3rb-media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav +2024-05-13 21:48:30,574 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/246g1564-media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav +2024-05-13 21:48:30,644 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ppnuqlwy-media/audio/Speech samples/eval_381_492597073098578f0605.wav +2024-05-13 21:48:30,660 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/he6ljg25-media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav +2024-05-13 21:48:31,008 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep381 - {'id': 'QXJ0aWZhY3Q6ODM0ODA1OTg0', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI2Mzg5OA==', 'latestArtifact': None}} +2024-05-13 21:48:31,008 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,009 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json with policy now +2024-05-13 21:48:31,009 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:48:31,009 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:48:31,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:48:31,010 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav with policy now +2024-05-13 21:48:31,010 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav with policy now +2024-05-13 21:48:31,010 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav with policy now +2024-05-13 21:48:31,010 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav with policy now +2024-05-13 21:48:31,011 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,011 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_492597073098578f0605.wav with policy now +2024-05-13 21:48:31,011 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,011 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav with policy now +2024-05-13 21:48:31,011 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,011 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav with policy now +2024-05-13 21:48:31,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:48:31,457 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/2f8ll71e-media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json +2024-05-13 21:48:32,025 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/t0g9u7ur-media/audio/Speech samples/eval_381_492597073098578f0605.wav +2024-05-13 21:48:32,135 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/4xjdyiu6-media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav +2024-05-13 21:48:32,141 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/l91pm89r-media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav +2024-05-13 21:48:32,147 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8kqbn4sy-media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav +2024-05-13 21:48:32,148 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/cztr9kak-media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav +2024-05-13 21:48:32,153 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8ugzxajm-media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav +2024-05-13 21:48:32,178 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/mbhusgri-media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav +2024-05-13 21:48:32,258 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:32,794 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:32,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:33,259 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:34,013 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:35,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:35,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:37,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:48:37,135 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:48:38,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:38,085 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:39,358 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:40,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:40,679 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:43,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:43,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:44,267 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:44,397 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:46,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:46,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:48,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:48,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:49,627 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:49,783 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:48:50,271 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:51,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:51,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:52,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:48:52,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:48:52,272 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:54,636 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:59,421 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:48:59,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:59,468 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:59,637 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:01,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:01,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:04,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:04,638 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:04,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:07,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:49:07,135 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:49:07,283 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:07,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:09,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:09,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:10,329 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:10,860 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:49:10,861 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:49:10,862 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:49:10,862 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:49:11,285 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:49:12,286 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:12,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:12,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:13,286 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:15,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:15,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:15,863 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:18,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:18,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:19,786 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:49:20,293 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:20,634 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:20,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:21,788 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:22,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:49:22,135 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:49:23,259 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:23,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:25,903 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:25,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:26,297 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:27,086 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:28,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:28,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:30,299 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:31,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:31,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:32,301 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:33,026 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:33,098 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:49:33,099 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:49:33,099 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:49:33,099 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:49:33,301 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:49:33,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:33,666 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:34,302 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:35,303 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:36,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:36,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:37,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:49:37,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:49:38,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:43,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:44,215 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:49:45,490 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:45,517 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:48,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:48,219 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:48,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:49,789 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:49:50,846 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:50,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:52,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:49:52,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:49:52,314 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:53,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:53,488 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:54,312 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:56,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:56,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:58,860 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:58,880 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:59,313 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:00,320 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:01,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:01,580 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:04,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:04,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:04,518 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:06,324 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:06,764 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:06,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:07,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:50:07,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:50:09,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:09,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:09,685 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:12,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:12,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:12,328 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:12,785 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:50:12,787 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:50:12,788 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:50:12,788 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:50:13,328 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:13,328 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:50:14,329 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:14,789 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:14,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:14,917 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:16,330 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:17,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:17,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:19,791 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:50:19,793 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:20,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:20,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:22,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:50:22,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:50:22,962 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:22,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:25,340 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:28,338 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:30,426 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:31,105 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:50:32,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:32,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:34,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:34,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:35,428 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:37,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:50:37,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:50:37,384 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:37,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:40,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:40,074 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:40,346 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:40,514 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:42,823 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:42,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:45,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:45,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:46,076 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:46,351 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:48,231 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:48,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:49,773 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:50:49,773 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:50:49,774 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:50:49,774 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:50:49,793 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:50:50,354 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:50:51,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:51,032 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:51,355 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:51,794 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:52,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:50:52,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:50:52,356 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:53,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:53,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:54,357 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:56,212 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:56,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:56,358 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:57,385 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:59,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:59,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:01,842 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:01,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:02,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:04,363 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:04,396 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:04,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:07,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:07,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:07,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:51:07,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:51:08,296 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:09,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:09,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:13,297 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:16,370 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:17,678 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:51:18,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:18,853 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:18,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:19,796 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:51:21,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:21,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:21,711 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:51:21,712 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:51:21,712 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:51:21,713 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:51:22,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:51:22,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:51:22,373 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:22,374 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:51:23,374 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:24,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:24,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:24,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:26,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:26,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:28,377 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:29,413 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:29,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:29,533 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:32,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:32,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:32,379 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:34,380 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:34,535 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:34,643 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:34,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:36,381 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:37,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:51:37,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:51:37,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:37,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:40,002 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:40,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:40,303 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:42,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:42,740 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:45,305 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:45,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:45,448 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:47,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:47,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:49,798 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:51:50,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:50,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:50,799 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:52,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:51:52,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:51:52,438 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:52,459 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:52,658 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:51:52,659 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:51:52,659 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:51:52,660 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:51:53,391 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:53,391 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:51:54,391 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:54,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:54,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:56,661 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:02,396 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:02,655 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:03,119 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:52:03,681 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:03,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:06,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:06,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:07,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:52:07,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:52:08,329 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:08,399 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:09,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:09,397 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:11,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:11,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:12,401 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:13,586 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:14,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:14,382 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:14,402 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:15,662 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:52:15,663 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:52:15,663 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:52:15,663 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:52:16,404 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:16,404 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:52:16,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:16,880 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:17,404 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:18,664 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:19,290 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:19,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:19,801 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:52:21,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:21,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:22,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:52:22,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:52:24,288 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:24,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:24,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:26,824 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:26,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:29,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:29,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:29,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:31,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:31,820 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:34,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:34,283 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:34,414 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:35,075 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:36,693 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:36,720 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:37,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:52:37,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:52:38,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:39,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:40,359 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:42,419 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:45,879 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:46,404 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:52:47,653 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:47,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:48,422 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:49,803 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:52:50,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:50,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:51,500 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:52,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:52:52,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:52:52,424 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:52,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:52,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:53,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:52:53,363 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:52:53,364 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:52:53,364 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:52:53,425 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:52:54,425 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:55,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:55,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:55,426 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:56,426 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:57,456 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:57,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:57,658 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:00,275 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:00,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:02,457 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:02,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:02,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:05,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:05,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:07,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:53:07,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:53:07,817 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:08,395 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:08,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:10,433 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:11,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:11,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:12,818 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:13,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:13,872 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:16,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:16,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:18,714 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:19,193 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:19,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:19,805 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:53:20,438 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:21,908 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:21,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:22,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:53:22,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:53:24,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:24,528 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:24,710 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:26,441 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:27,934 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:53:27,936 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:53:27,936 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:53:27,936 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:53:28,443 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:53:29,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:29,937 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:30,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:32,312 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:53:32,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:32,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:32,445 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:34,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:34,848 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:34,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:35,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:37,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:53:37,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:53:37,562 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:37,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:40,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:40,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:41,327 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:42,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:42,886 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:44,450 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:45,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:45,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:47,260 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:48,148 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:48,168 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:49,808 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:53:50,761 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:50,797 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:52,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:53:52,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:53:52,304 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:53,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:53,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:56,157 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:56,180 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:57,624 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:58,458 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:58,889 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:58,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:01,071 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:01,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:02,625 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:03,811 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:54:03,812 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:54:03,812 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:54:03,813 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:54:03,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:03,943 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:04,461 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:54:05,462 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:06,462 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:06,631 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:06,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:07,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:54:07,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:54:08,314 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:09,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:09,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:10,466 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:13,744 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:14,468 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:17,505 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:54:17,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:17,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:18,471 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:18,882 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:19,810 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:54:20,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:20,201 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:20,472 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:22,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:54:22,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:54:22,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:22,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:24,274 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:25,566 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:25,593 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:28,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:28,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:29,275 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:30,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:30,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:33,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:33,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:34,276 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:36,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:36,356 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:37,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:54:37,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:54:37,390 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:54:37,390 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:54:37,391 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:54:37,391 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:54:37,481 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:54:38,481 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:39,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:39,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:39,392 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:39,481 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:41,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:41,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:44,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:44,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:44,392 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:46,484 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:47,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:47,037 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:49,643 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:49,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:49,812 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:54:49,813 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:52,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:54:52,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:54:52,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:52,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:52,487 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:54,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:54,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:54,961 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:56,489 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:58,490 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:58,843 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:54:58,844 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:54:58,844 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:54:58,845 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:54:59,491 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:59,491 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:55:00,491 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:00,845 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:02,667 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:55:03,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:03,964 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:05,846 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:06,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:06,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:07,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:55:07,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:55:09,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:09,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:11,302 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:12,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:12,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:14,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:14,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:16,501 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:17,034 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:17,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:17,627 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:19,815 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:55:20,232 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:20,260 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:22,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:55:22,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:22,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:55:22,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:22,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:25,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:25,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:26,507 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:27,260 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:28,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:28,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:30,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:30,666 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:32,510 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:32,544 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:33,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:33,322 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:35,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:35,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:36,513 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:37,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:55:37,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:55:37,879 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:37,935 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:55:37,936 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:55:37,936 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:55:37,936 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:55:38,514 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:55:38,542 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:38,568 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:39,514 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:40,515 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:41,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:41,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:42,516 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:43,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:48,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:49,241 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:55:49,818 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:55:50,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:50,292 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:52,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:55:52,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:55:52,903 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:52,931 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:53,354 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:54,524 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:55,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:55,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:58,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:58,105 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:58,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:00,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:00,713 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:03,378 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:03,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:03,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:06,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:06,030 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:06,531 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:07,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:56:07,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:56:08,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:08,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:09,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:11,534 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:11,560 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:12,535 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:14,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:14,224 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:14,869 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:15,632 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:56:15,634 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:56:15,634 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:56:15,634 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:56:16,538 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:16,538 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:56:16,816 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:16,842 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:17,538 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:19,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:19,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:19,819 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:56:20,540 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:20,821 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:21,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:22,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:22,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:56:22,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:56:22,542 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:24,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:24,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:26,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:27,311 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:27,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:32,247 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:32,549 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:35,272 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:56:36,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:36,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:37,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:56:37,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:56:37,332 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:39,174 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:39,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:41,889 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:41,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:42,333 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:44,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:44,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:46,556 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:47,337 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:47,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:47,356 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:49,822 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:56:49,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:49,991 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:51,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:56:51,387 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:56:51,387 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:56:51,388 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:56:51,559 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:56:52,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:56:52,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:56:52,479 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:52,560 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:52,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:52,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:53,560 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:55,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:55,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:57,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:58,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:58,290 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:58,563 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:00,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:00,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:02,566 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:03,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:03,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:03,355 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:04,567 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:05,833 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:05,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:06,568 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:07,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:57:07,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:57:08,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:08,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:09,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:11,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:11,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:14,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:14,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:14,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:19,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:19,825 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:57:21,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:57:21,695 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:57:21,695 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:57:21,696 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:57:22,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:57:22,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:57:22,344 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:57:22,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:22,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:57:23,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:23,438 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:23,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:24,335 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:26,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:26,061 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:28,618 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:28,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:29,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:31,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:31,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:32,583 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:33,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:33,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:34,917 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:36,586 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:36,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:36,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:37,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:57:37,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:57:39,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:39,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:40,588 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:40,665 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:41,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:41,891 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:44,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:44,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:44,591 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:44,659 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:57:44,659 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:57:44,660 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:57:44,660 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:57:45,592 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:45,592 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:57:46,593 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:46,661 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:47,089 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:47,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:49,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:49,772 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:49,828 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:57:51,829 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:52,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:57:52,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:57:52,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:52,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:55,037 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:55,069 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:57,263 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:57,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:57,671 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:00,339 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:00,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:02,399 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:02,602 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:07,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:58:07,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:58:08,311 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:08,457 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:58:08,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:08,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:10,607 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:11,234 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:11,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:13,629 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:13,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:13,889 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:16,610 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:16,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:16,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:19,472 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:19,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:19,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:19,830 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:58:20,613 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:21,681 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:58:21,682 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:58:21,682 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:58:21,682 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:58:22,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:58:22,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:58:22,214 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:22,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:22,614 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:22,614 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:58:23,615 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:24,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:24,773 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:24,949 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:26,617 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:27,541 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:27,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:29,950 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:30,188 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:30,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:32,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:32,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:34,950 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:35,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:35,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:37,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:58:37,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:58:38,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:38,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:38,623 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:40,350 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:40,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:40,830 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:43,485 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:43,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:45,351 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:46,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:46,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:48,629 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:49,832 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:58:50,833 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:52,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:58:52,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:58:52,631 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:55,454 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:58:55,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:55,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:55,917 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:55,980 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:58:55,981 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:58:55,981 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:58:55,982 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:58:56,633 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:58:57,634 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:58,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:58,068 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:58,635 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:00,637 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:00,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:00,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:01,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:02,638 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:03,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:03,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:06,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:06,201 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:06,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:07,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:59:07,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:59:08,848 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:08,874 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:11,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:11,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:11,996 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:12,643 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:14,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:14,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:16,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:16,997 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:17,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:19,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:19,736 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:19,836 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:59:22,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:22,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:59:22,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:59:22,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:22,451 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:25,136 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:25,159 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:26,651 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:27,272 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:27,700 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:27,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:30,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:30,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:31,647 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:59:31,647 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:59:31,648 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:59:31,648 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:59:31,655 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:59:32,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:32,656 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:33,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:33,122 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:33,657 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:37,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:59:37,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:59:38,360 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:38,660 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:41,223 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:59:42,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:42,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:42,663 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:43,934 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:44,664 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:45,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:45,071 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:46,665 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:47,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:47,721 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:49,837 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:59:49,839 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:50,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:50,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:52,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:59:52,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:59:53,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:53,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:55,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:55,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:56,021 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:58,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:58,802 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:00,337 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:01,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:01,427 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:03,577 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:00:03,578 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:00:03,578 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:00:03,578 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:00:03,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:00:04,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:04,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:04,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:05,579 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:05,676 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:06,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:06,771 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:07,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:00:07,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:00:09,432 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:09,451 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:11,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:12,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:12,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:12,680 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:14,932 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:14,946 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:16,433 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:17,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:17,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:18,684 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:19,840 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:00:20,292 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:20,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:22,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:22,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:00:22,141 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:00:22,686 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:24,688 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:24,870 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:00:24,872 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:00:24,872 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:00:24,872 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:00:25,688 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:25,689 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:00:26,690 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:27,873 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:28,043 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:00:28,284 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:29,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:32,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:32,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:32,874 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:34,810 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:34,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:37,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:00:37,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:00:37,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:37,603 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:38,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:40,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:40,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:42,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:43,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:43,132 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:44,158 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:45,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:45,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:48,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:48,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:49,159 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:49,842 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:00:51,547 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:51,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:52,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:00:52,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:00:52,706 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:54,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:54,178 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:54,335 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:56,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:56,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:58,710 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:59,396 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:59,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:00,095 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:02,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:02,051 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:02,712 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:03,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:01:03,185 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:01:03,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:01:03,186 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:01:03,712 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:01:04,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:04,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:04,713 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:05,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:05,714 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:06,714 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:07,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:01:07,141 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:01:07,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:07,309 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:10,301 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:15,171 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:01:15,201 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:15,216 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:15,302 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:17,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:17,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:19,845 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:01:20,524 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:20,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:20,723 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:20,846 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:22,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:01:22,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:01:23,075 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:23,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:25,603 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:25,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:26,309 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:28,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:28,276 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:30,730 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:31,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:31,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:31,952 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:33,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:33,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:36,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:36,463 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:36,733 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:37,038 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:37,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:01:37,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:01:39,132 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:39,153 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:39,409 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:01:40,218 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:01:40,218 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:01:40,218 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:01:40,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:40,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:01:41,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:41,781 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:41,800 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:42,219 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:44,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:44,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:44,739 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:46,740 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:46,941 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:46,979 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:47,873 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:49,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:49,631 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:49,848 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:01:52,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:01:52,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:01:52,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:52,216 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:53,310 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:54,745 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:58,462 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:59,938 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:01:59,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:59,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:02,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:02,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:03,462 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:05,275 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:05,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:06,752 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:07,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:02:07,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:02:08,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:08,046 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:09,302 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:10,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:10,778 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:12,629 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:02:12,630 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:02:12,630 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:02:12,631 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:02:12,756 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:02:13,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:13,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:13,757 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:14,631 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:14,757 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:16,185 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:16,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:18,759 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:18,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:18,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:19,850 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:02:19,851 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:21,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:21,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:22,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:02:22,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:02:22,762 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:23,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:23,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:24,763 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:25,610 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:26,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:26,219 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:28,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:28,515 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:30,657 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:30,767 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:30,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:30,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:32,768 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:33,228 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:33,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:35,711 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:35,741 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:36,000 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:37,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:02:37,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:02:41,369 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:42,961 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:02:43,000 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:43,014 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:45,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:45,395 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:46,370 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:47,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:47,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:49,853 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:02:50,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:50,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:51,854 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:52,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:02:52,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:02:52,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:52,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:54,993 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:55,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:57,364 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:57,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:57,404 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:58,783 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:59,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:59,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:00,785 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:02,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:02,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:02,787 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:02,788 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:03,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 22:03:03,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 22:03:03,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:03:03,189 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:03:03,191 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:03:03,788 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav +2024-05-13 22:03:03,788 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json +2024-05-13 22:03:03,788 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav +2024-05-13 22:03:03,788 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 22:03:03,987 INFO wandb-upload_6:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmp2ug83zl8 +2024-05-13 22:03:04,228 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/skxedlc6-media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json +2024-05-13 22:03:04,719 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep508 - {'id': 'QXJ0aWZhY3Q6ODM0ODM0NzUy', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI4MTIzMA==', 'latestArtifact': None}} +2024-05-13 22:03:04,719 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,719 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json with policy now +2024-05-13 22:03:04,720 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:03:04,720 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:03:04,721 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:03:04,721 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,721 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav with policy now +2024-05-13 22:03:04,721 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,721 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav with policy now +2024-05-13 22:03:04,721 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,721 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav with policy now +2024-05-13 22:03:04,721 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,721 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav with policy now +2024-05-13 22:03:04,723 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,723 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav with policy now +2024-05-13 22:03:04,723 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,723 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav with policy now +2024-05-13 22:03:04,723 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,723 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav with policy now +2024-05-13 22:03:04,745 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:04,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:04,788 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:03:04,862 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/viy58bog-media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav +2024-05-13 22:03:05,025 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/31xq2da1-media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav +2024-05-13 22:03:05,041 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ffnsvkzb-media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav +2024-05-13 22:03:05,055 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/k5bltd4d-media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav +2024-05-13 22:03:05,065 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/d1n4r5hq-media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav +2024-05-13 22:03:05,071 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/1xe9kgz5-media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav +2024-05-13 22:03:05,081 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/r4b78mdn-media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav +2024-05-13 22:03:05,156 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8m07u8eb-media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json +2024-05-13 22:03:05,790 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:05,978 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/cw1x5ci0-media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav +2024-05-13 22:03:06,164 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8m00leil-media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav +2024-05-13 22:03:06,180 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/aib73xk5-media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav +2024-05-13 22:03:06,183 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/sn8slo2m-media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav +2024-05-13 22:03:06,223 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/5yii1mxx-media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav +2024-05-13 22:03:06,790 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:07,073 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/jfvmb17v-media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav +2024-05-13 22:03:07,122 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/44m4arab-media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav +2024-05-13 22:03:07,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:03:07,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:03:07,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:07,370 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:08,349 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:09,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:09,878 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:12,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:12,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:13,350 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:15,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:15,329 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:17,995 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:18,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:18,351 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:19,855 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:03:21,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:03:21,133 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:03:21,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:03:21,709 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:03:21,802 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:03:22,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:03:22,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:03:22,803 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:23,804 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:24,342 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:25,736 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:03:25,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:25,787 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:26,806 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:28,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:28,166 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:28,807 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:30,252 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:31,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:31,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:33,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:33,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:35,253 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:36,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:36,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:37,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:03:37,144 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:03:39,021 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:39,050 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:40,347 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:41,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:41,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:44,403 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:44,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:45,347 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:47,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:47,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:48,820 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:49,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:49,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:49,858 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:03:50,859 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:52,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:03:52,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:03:52,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:52,335 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:54,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:55,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:56,486 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:56,826 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:57,633 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:57,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:00,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:00,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:00,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:04:01,000 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:04:01,000 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:04:01,001 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:04:01,829 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:01,829 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:04:02,001 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:02,830 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:02,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:02,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:06,833 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:07,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:04:07,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:04:07,369 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:10,836 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:11,047 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:04:11,084 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:11,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:12,635 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:12,838 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:13,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:13,541 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:16,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:16,148 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:17,636 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:18,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:18,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:19,860 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:04:21,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:21,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:22,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:04:22,145 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:04:23,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:24,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:24,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:26,732 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:26,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:28,292 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:28,850 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:29,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:29,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:32,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:32,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:33,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:34,690 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:34,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:36,962 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:04:36,963 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:04:36,964 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:04:36,964 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:04:37,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:04:37,145 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:04:37,328 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:37,346 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:37,857 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:37,858 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:04:38,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:38,858 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:40,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:40,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:42,756 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:42,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:43,764 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:44,862 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:45,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:45,611 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:48,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:48,309 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:48,865 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:49,006 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:49,863 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:04:50,867 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:52,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:04:52,145 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:04:54,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:54,870 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:55,947 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:04:55,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:55,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:58,497 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:58,511 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:59,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:01,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:01,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:03,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:03,879 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:04,354 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:05,710 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:05:05,711 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:05:05,712 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:05:05,712 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:05:05,879 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:05:06,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:06,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:06,879 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:07,145 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:05:07,146 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:05:07,880 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:09,284 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:09,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:09,355 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:12,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:12,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:14,356 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:14,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:14,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:16,886 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:17,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:17,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:19,613 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:19,865 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:05:20,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:20,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:22,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:05:22,145 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:05:22,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:22,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:22,889 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:25,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:25,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:25,393 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:27,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:27,951 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:28,893 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:30,425 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:30,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:30,507 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:30,894 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:31,778 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:05:31,780 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:05:31,780 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:05:31,780 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:05:31,895 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:05:32,815 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:32,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:32,896 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:33,896 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:35,781 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:37,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:05:37,145 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:05:39,813 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:05:39,847 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:39,859 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:40,957 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:42,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:42,208 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:42,903 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:44,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:44,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:45,958 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:47,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:47,063 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:49,468 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:49,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:49,867 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:05:51,869 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:51,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:51,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:52,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:05:52,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:05:54,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:54,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:56,774 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:56,797 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:56,913 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:57,369 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:59,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:59,253 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:01,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:01,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:02,485 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:02,916 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:04,146 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:04,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:06,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:06,622 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:07,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:06:07,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:06:08,286 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:08,920 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:09,045 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:09,067 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:10,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:06:10,147 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:06:10,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:06:10,148 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:06:10,922 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:06:11,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:11,490 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:11,922 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:12,923 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:13,728 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:13,880 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:13,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:14,924 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:16,926 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:19,683 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:19,869 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:06:21,399 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:06:22,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:22,021 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:22,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:06:22,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:06:24,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:24,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:25,283 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:27,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:27,129 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:29,780 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:29,823 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:30,284 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:32,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:32,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:35,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:35,224 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:35,583 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:36,939 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:37,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:06:37,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:06:37,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:37,977 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:40,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:40,634 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:41,319 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:43,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:43,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:44,945 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:45,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:45,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:46,365 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:47,762 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:06:47,763 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:06:47,763 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:06:47,764 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:06:47,948 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:06:48,176 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:48,187 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:48,949 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:49,873 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:06:49,949 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:50,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:50,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:51,536 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:52,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:06:52,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:06:52,951 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:53,425 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:53,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:56,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:56,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:56,954 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:57,325 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:58,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:58,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:58,955 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:02,326 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:06,826 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:07:06,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:06,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:07,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:07:07,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:07:07,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:09,490 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:09,520 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:12,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:12,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:12,965 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:13,309 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:14,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:14,810 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:17,373 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:17,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:18,310 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:19,874 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:07:20,033 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:07:20,034 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:07:20,034 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:07:20,034 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:07:20,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:20,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:20,970 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:20,970 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:07:21,970 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:22,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:07:22,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:07:22,741 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:22,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:23,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:25,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:25,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:26,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:28,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:28,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:28,683 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:30,976 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:30,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:31,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:33,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:33,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:33,803 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:34,979 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:36,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:36,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:36,981 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:37,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:07:37,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:07:38,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:38,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:39,331 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:41,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:41,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:44,156 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:44,170 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:44,332 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:49,276 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:07:49,277 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:07:49,277 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:07:49,277 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:07:49,876 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:07:49,877 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:49,990 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:07:50,991 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:51,873 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:07:51,992 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:52,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:07:52,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:07:52,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:53,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:55,335 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:55,864 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:55,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:58,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:58,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:00,729 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:01,001 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:01,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:01,316 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:03,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:04,004 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:05,730 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:06,739 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:06,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:07,005 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:07,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:08:07,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:08:09,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:09,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:11,603 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:11,946 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:11,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:13,010 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:14,524 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:14,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:15,012 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:16,194 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:08:16,195 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:08:16,195 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:08:16,195 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:08:17,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:17,013 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:08:17,032 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:17,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:18,014 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:19,015 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:19,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:19,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:19,880 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:08:22,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:08:22,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:08:22,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:22,265 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:22,291 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:24,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:24,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:27,365 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:27,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:27,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:29,022 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:30,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:30,042 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:32,367 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:37,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:08:37,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:08:37,371 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:37,948 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:08:37,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:37,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:40,586 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:40,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:41,030 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:43,169 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:43,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:43,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:45,997 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:46,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:47,034 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:48,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:48,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:48,928 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:49,882 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:08:51,435 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:51,457 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:52,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:08:52,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:08:53,039 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:53,983 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:54,041 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:08:54,042 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:08:54,042 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:08:54,042 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:08:54,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:54,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:55,044 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:55,044 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:08:56,045 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:56,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:56,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:59,047 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:59,334 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:59,335 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:59,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:01,048 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:01,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:02,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:04,335 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:04,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:04,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:07,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:09:07,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:09:07,365 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:07,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:09,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:10,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:10,233 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:12,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:12,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:14,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:15,403 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:15,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:17,059 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:19,413 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:19,884 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:09:22,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:09:22,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:09:23,144 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:09:23,171 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:23,188 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:24,833 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:25,065 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:25,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:25,802 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:28,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:28,469 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:28,966 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:09:28,967 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:09:28,967 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:09:28,968 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:09:29,067 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:09:29,968 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:30,068 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:31,068 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:31,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:31,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:33,747 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:33,764 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:35,071 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:35,391 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:36,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:36,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:37,072 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:37,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:09:37,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:09:38,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:39,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:39,073 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:41,380 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:41,643 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:41,661 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:44,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:44,327 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:46,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:46,899 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:46,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:49,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:49,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:49,887 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:09:51,889 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:52,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:52,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:09:52,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:09:52,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:54,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:54,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:55,084 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:57,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:57,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:57,703 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:00,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:00,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:02,275 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:10:02,276 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:10:02,276 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:10:02,277 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:10:03,090 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:10:03,277 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:04,091 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:05,092 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:07,148 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:10:07,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:10:07,929 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:10:07,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:07,977 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:09,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:10,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:10,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:11,095 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:13,353 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:13,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:14,473 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:15,098 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:15,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:15,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:17,099 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:18,408 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:18,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:19,101 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:19,732 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:19,889 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:10:20,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:21,000 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:22,148 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:10:22,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:10:23,670 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:23,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:25,332 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:26,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:26,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:29,180 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:29,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:30,333 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:31,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:31,873 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:32,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:10:32,347 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:10:32,347 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:10:32,347 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:10:33,112 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:10:34,113 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:34,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:34,492 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:35,113 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:35,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:37,148 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:10:37,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:10:37,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:37,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:39,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:39,871 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:40,369 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:42,512 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:42,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:45,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:45,120 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:45,134 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:45,775 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:49,893 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:10:50,894 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:51,130 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:52,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:10:52,150 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:10:53,054 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:10:53,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:53,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:55,617 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:55,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:56,271 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:57,136 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:58,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:58,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:59,138 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:59,809 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:10:59,810 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:10:59,810 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:10:59,810 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:11:00,139 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:11:00,857 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:00,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:01,140 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:01,811 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:02,142 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:03,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:03,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:06,316 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:06,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:06,812 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:07,148 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:11:07,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:11:09,071 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:09,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:11,149 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:11,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:11,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:12,404 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:14,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:14,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:17,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:17,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:17,405 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:19,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:19,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:19,895 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:11:22,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:11:22,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:11:22,208 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:22,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:23,160 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:23,359 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:24,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:24,979 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:27,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:27,586 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:28,359 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:30,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:30,215 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:31,168 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:33,793 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:35,172 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:37,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:11:37,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:11:37,261 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:11:37,343 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:11:37,343 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:11:37,344 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:11:38,175 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:11:39,176 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:39,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:39,556 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:11:40,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:40,557 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:40,580 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:41,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:43,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:43,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:43,179 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:44,659 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:45,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:45,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:48,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:48,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:49,660 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:49,897 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:11:51,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:51,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:52,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:11:52,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:11:53,684 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:53,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:55,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:56,335 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:56,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:58,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:59,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:01,197 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:01,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:01,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:01,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:04,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:04,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:06,325 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:07,085 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:07,100 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:07,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:12:07,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:12:09,204 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:09,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:09,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:11,504 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:12,288 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:12,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:13,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:12:13,137 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:12:13,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:12:13,138 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:12:13,208 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:12:14,209 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:14,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:14,891 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:15,210 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:16,930 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:17,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:17,570 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:19,214 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:19,900 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:12:21,216 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:22,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:12:22,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:12:22,296 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:23,218 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:25,334 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:12:25,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:25,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:27,297 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:27,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:27,997 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:30,757 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:30,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:32,298 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:33,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:33,395 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:36,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:36,126 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:37,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:12:37,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:12:37,321 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:38,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:38,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:39,235 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:41,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:41,358 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:42,321 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:43,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:43,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:45,677 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:12:45,678 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:12:45,678 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:12:45,678 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:12:46,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:12:46,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:46,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:47,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:47,679 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:48,244 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:49,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:49,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:49,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:12:51,972 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:51,994 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:52,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:12:52,150 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:12:53,248 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:53,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:54,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:54,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:57,253 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:57,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:57,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:58,976 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:59,254 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:00,028 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:00,049 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:01,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:02,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:02,710 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:04,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:07,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:13:07,150 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:13:09,327 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:10,552 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:13:10,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:10,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:13,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:13,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:13,349 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:13:13,350 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:13:13,351 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:13:13,351 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:13:14,269 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:13:14,352 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:15,270 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:15,871 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:15,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:16,272 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:18,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:18,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:19,352 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:19,905 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:13:21,218 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:21,240 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:22,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:13:22,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:13:23,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:23,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:24,279 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:24,899 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:26,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:26,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:29,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:29,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:30,095 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:30,285 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:31,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:31,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:34,289 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:34,533 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:34,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:35,456 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:37,138 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:37,149 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:37,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:13:37,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:13:38,293 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:39,185 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:13:39,186 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:13:39,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:13:39,187 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:13:39,294 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:39,294 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:13:39,651 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:39,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:40,295 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:41,188 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:42,175 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:42,198 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:44,746 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:44,776 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:46,189 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:47,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:47,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:49,907 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:13:50,304 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:51,909 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:52,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:13:52,150 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:13:55,326 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:13:55,364 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:55,386 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:57,331 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:57,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:58,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:00,722 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:00,739 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:02,316 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:02,916 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:03,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:03,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:06,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:06,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:07,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:14:07,150 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:14:08,321 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:08,360 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:08,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:08,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:11,508 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:11,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:12,325 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:13,896 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:14,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:14,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:15,418 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:14:15,419 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:14:15,419 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:14:15,420 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:14:16,329 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:16,329 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:14:16,756 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:16,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:17,330 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:18,331 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:19,330 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:19,353 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:19,675 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:19,910 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:14:20,333 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:21,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:21,941 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:22,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:14:22,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:14:24,622 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:24,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:25,342 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:27,279 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:27,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:30,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:30,061 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:30,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:32,713 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:32,733 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:35,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:37,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:14:37,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:14:40,353 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:40,553 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:14:40,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:40,614 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:40,678 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:43,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:43,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:45,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:45,719 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:45,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:48,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:48,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:49,912 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:14:50,364 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:50,913 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:51,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:51,046 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:52,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:14:52,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:14:53,756 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:14:53,757 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:14:53,757 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:14:53,757 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:14:53,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:53,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:54,368 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:54,368 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:14:55,369 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:56,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:56,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:56,758 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:58,372 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:59,021 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:59,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:01,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:01,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:02,376 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:02,480 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:04,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:04,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:04,378 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:06,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:06,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:07,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:15:07,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:15:08,284 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:09,386 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:09,418 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:11,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:12,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:13,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:14,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:14,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:17,170 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:17,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:18,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:19,915 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:15:20,393 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:22,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:15:22,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:15:23,328 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:25,024 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:15:25,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:25,074 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:27,739 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:15:27,740 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:15:27,740 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:15:27,740 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:15:27,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:27,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:28,401 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:28,401 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:15:28,741 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:29,402 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:30,463 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:30,484 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:33,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:33,248 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:34,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:34,407 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:35,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:35,874 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:37,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:15:37,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:15:38,411 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:38,457 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:38,482 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:39,436 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:41,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:41,042 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:42,415 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:43,618 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:43,642 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:44,417 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:44,651 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:46,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:46,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:49,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:49,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:49,652 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:49,917 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:15:51,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:51,812 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:52,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:15:52,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:15:54,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:54,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:55,251 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:55,433 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:15:55,434 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:15:55,434 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:15:55,613 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:15:56,430 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:56,430 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:15:57,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:57,185 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:57,431 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:59,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:59,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:00,614 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:02,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:02,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:05,616 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:06,439 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:07,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:16:07,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:16:10,041 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:16:10,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:10,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:11,283 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:12,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:13,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:13,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:15,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:15,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:16,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:18,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:18,333 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:18,449 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:19,920 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:16:20,451 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:20,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:20,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:21,970 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:22,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:16:22,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:16:22,476 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:16:22,477 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:16:22,477 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:16:22,478 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:16:23,453 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:23,453 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:16:23,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:23,593 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:24,454 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:26,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:26,178 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:27,478 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:28,847 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:28,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:31,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:31,627 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:32,459 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:33,141 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:34,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:34,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:37,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:37,064 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:37,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:16:37,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:16:38,380 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:39,693 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:39,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:42,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:42,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:43,381 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:45,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:45,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:46,470 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:47,702 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:47,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:48,870 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:49,922 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:16:52,152 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:16:52,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:16:52,475 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:54,307 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:56,951 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:16:56,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:57,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:58,481 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:59,554 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:59,681 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:59,700 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:00,001 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:17:00,002 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:17:00,002 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:17:00,002 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:17:00,482 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:00,483 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:17:01,483 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:02,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:02,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:04,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:04,424 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:05,003 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:06,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:06,732 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:07,152 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:17:07,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:17:08,493 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:09,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:09,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:10,077 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:11,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:11,586 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:13,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:14,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:15,078 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:16,487 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:16,512 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:18,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:18,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:19,924 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:17:20,926 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:21,388 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:21,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:22,153 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:17:22,154 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:17:23,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:23,836 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:26,242 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:26,259 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:26,352 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:28,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:28,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:31,291 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:31,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:31,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:36,517 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:36,602 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:37,152 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:17:37,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:17:38,518 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:38,782 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:17:38,802 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:38,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:40,519 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:41,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:41,214 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:41,469 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 22:17:41,469 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 22:17:41,476 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:17:41,520 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json +2024-05-13 22:17:41,520 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 22:17:41,530 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:17:41,531 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:17:41,942 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/xavg6oni-media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json +2024-05-13 22:17:42,240 INFO wandb-upload_0:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmp96rb8gl7 +2024-05-13 22:17:42,521 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav +2024-05-13 22:17:42,521 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 22:17:42,875 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep635 - {'id': 'QXJ0aWZhY3Q6ODM0ODUxMzg4', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI4Njc5MA==', 'latestArtifact': None}} +2024-05-13 22:17:42,880 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,882 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:42,882 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json with policy now +2024-05-13 22:17:42,895 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,900 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav with policy now +2024-05-13 22:17:42,901 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,901 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav with policy now +2024-05-13 22:17:42,901 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav with policy now +2024-05-13 22:17:42,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav with policy now +2024-05-13 22:17:42,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav with policy now +2024-05-13 22:17:42,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,903 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav with policy now +2024-05-13 22:17:42,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,903 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav with policy now +2024-05-13 22:17:43,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:43,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:43,740 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:43,751 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/4ai8emmm-media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json +2024-05-13 22:17:43,853 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/woueedf5-media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav +2024-05-13 22:17:43,979 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/h2ochj19-media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav +2024-05-13 22:17:44,169 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/3gsxo8k2-media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav +2024-05-13 22:17:44,263 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/smdoboj8-media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav +2024-05-13 22:17:44,282 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/sxh3z35v-media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav +2024-05-13 22:17:44,303 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/9hmhf2xq-media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav +2024-05-13 22:17:44,314 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8bfu8i90-media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav +2024-05-13 22:17:44,525 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:44,959 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/315gc28l-media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav +2024-05-13 22:17:45,381 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/vn2mvetc-media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav +2024-05-13 22:17:45,406 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/cxek704t-media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav +2024-05-13 22:17:45,416 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/3qpf0vlv-media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav +2024-05-13 22:17:46,060 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/cmgjfhdc-media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav +2024-05-13 22:17:46,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:46,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:46,441 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/h55dzwfh-media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav +2024-05-13 22:17:46,484 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/zse5ms4e-media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav +2024-05-13 22:17:46,527 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:48,229 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:48,528 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:48,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:49,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:49,927 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:17:51,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:51,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:52,152 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:17:52,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:17:53,303 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:54,290 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:54,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:56,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:57,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:58,304 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:59,757 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:59,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:02,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:02,542 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:02,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:03,834 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:05,278 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:05,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:06,544 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:07,021 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:18:07,023 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:18:07,023 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:18:07,023 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:18:07,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:18:07,260 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:18:07,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:07,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:18:07,873 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:07,899 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:08,546 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:09,417 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:10,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:10,830 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:13,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:13,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:14,418 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:16,094 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:16,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:19,418 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:19,930 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:18:22,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:18:22,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:18:23,098 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:18:24,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:24,166 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:25,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:26,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:26,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:28,562 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:29,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:29,622 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:31,277 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:32,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:32,442 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:35,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:35,105 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:36,278 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:37,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:18:37,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:18:37,781 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:37,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:38,570 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:40,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:40,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:41,360 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:42,573 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:43,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:43,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:45,641 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:45,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:46,577 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:47,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:48,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:48,266 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:48,979 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:18:48,980 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:18:48,980 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:18:48,981 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:18:49,579 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:49,579 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:18:49,932 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:18:50,580 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:50,937 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:50,955 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:52,169 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:52,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:18:52,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:18:52,581 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:53,481 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:53,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:56,118 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:56,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:57,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:58,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:58,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:01,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:01,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:02,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:07,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:19:07,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:19:07,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:08,594 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:09,572 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:19:09,598 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:09,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:12,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:12,267 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:12,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:14,879 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:14,905 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:16,600 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:17,476 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:17,516 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:17,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:19,936 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:19:19,979 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:19,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:22,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:19:22,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:19:22,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:22,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:22,604 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:23,340 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:24,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:24,589 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:25,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:19:25,139 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:19:25,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:19:25,139 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:19:25,607 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:25,607 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:19:26,608 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:27,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:27,028 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:28,670 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:29,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:29,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:30,612 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:31,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:31,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:32,614 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:33,908 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:34,364 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:34,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:36,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:36,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:37,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:19:37,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:19:39,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:39,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:39,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:41,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:41,820 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:44,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:44,294 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:44,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:46,626 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:49,729 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:49,937 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:19:51,560 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:19:52,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:19:52,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:19:52,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:52,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:55,012 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:55,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:55,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:56,635 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:57,901 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:57,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:00,013 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:00,382 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:00,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:01,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:20:01,345 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:20:01,345 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:20:01,347 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:20:01,639 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:20:02,640 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:02,884 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:02,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:03,641 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:05,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:05,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:05,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:06,643 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:07,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:20:07,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:20:07,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:07,598 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:09,948 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:09,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:10,646 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:11,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:12,435 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:12,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:12,647 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:15,119 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:15,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:16,509 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:17,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:17,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:19,939 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:20:20,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:20,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:22,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:22,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:20:22,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:20:22,653 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:23,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:23,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:25,990 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:26,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:27,312 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:28,693 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:28,711 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:32,313 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:34,510 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:20:34,511 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:20:34,512 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:20:34,512 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:20:34,661 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:20:35,662 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:36,260 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:20:36,663 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:37,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:20:37,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:20:37,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:37,358 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:37,374 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:40,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:40,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:42,635 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:42,646 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:42,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:42,667 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:45,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:45,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:47,771 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:48,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:48,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:48,670 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:49,942 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:20:50,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:50,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:52,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:20:52,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:20:52,673 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:53,218 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:53,535 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:53,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:54,674 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:55,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:20:55,286 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:20:55,286 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:20:55,286 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:20:55,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:55,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:20:56,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:56,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:56,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:58,287 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:58,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:58,782 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:01,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:01,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:03,288 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:03,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:03,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:06,520 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:06,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:07,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:21:07,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:21:08,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:09,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:09,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:12,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:12,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:14,017 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:14,687 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:14,832 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:14,856 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:19,019 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:19,945 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:21:22,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:21:22,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:21:22,693 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:23,081 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:21:24,107 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:24,118 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:24,398 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:26,696 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:26,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:26,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:29,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:29,432 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:30,245 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:30,698 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:31,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:32,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:33,167 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:21:33,169 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:21:33,169 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:21:33,169 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:21:33,700 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:33,700 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:21:34,514 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:34,537 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:34,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:36,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:36,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:37,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:37,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:37,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:21:37,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:21:39,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:39,730 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:41,364 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:42,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:42,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:45,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:45,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:46,364 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:47,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:47,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:49,947 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:21:50,517 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:50,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:51,809 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:52,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:21:52,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:21:52,713 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:53,171 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:53,187 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:55,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:55,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:57,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:58,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:58,492 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:01,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:01,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:02,718 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:02,924 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:07,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:22:07,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:22:08,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:08,724 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:09,839 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:22:10,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:10,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:10,570 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:22:10,571 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:22:10,572 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:22:10,572 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:22:10,725 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:22:11,726 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:12,727 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:12,877 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:12,890 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:13,426 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:14,728 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:15,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:15,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:16,730 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:18,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:18,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:18,592 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:19,950 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:22:20,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:20,824 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:22,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:22:22,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:22:23,479 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:23,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:24,297 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:26,188 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:26,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:28,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:28,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:28,878 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:29,556 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:31,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:31,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:34,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:34,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:34,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:36,739 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:36,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:37,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:22:37,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:22:38,742 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:39,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:39,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:40,314 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:41,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:42,012 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:43,145 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:22:43,146 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:22:43,146 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:22:43,146 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:22:43,746 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:43,746 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:22:44,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:44,710 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:44,747 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:46,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:47,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:47,417 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:48,749 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:49,952 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:22:51,821 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:52,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:22:52,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:22:52,752 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:54,753 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:55,126 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:22:55,146 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:55,158 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:57,249 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:57,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:57,860 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:00,480 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:00,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:02,250 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:03,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:03,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:04,762 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:05,846 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:05,863 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:07,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:23:07,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:23:07,389 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:08,506 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:08,526 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:11,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:11,271 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:12,390 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:13,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:13,952 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:15,409 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:23:15,411 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:23:15,411 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:23:15,412 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:23:15,769 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:23:16,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:16,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:16,770 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:17,412 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:17,771 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:19,397 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:19,438 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:19,955 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:23:22,168 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:22,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:23:22,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:23:22,186 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:22,775 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:23,330 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:24,814 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:24,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:27,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:27,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:28,779 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:29,089 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:30,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:30,088 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:32,663 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:32,680 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:32,782 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:34,784 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:34,961 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:35,009 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:23:35,011 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:23:35,011 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:23:35,011 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:23:35,784 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:35,785 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:23:36,786 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:37,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:23:37,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:23:40,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:40,406 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:23:40,438 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:40,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:43,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:43,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:45,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:45,722 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:45,735 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:48,435 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:48,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:49,957 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:23:50,958 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:51,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:51,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:52,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:23:52,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:23:52,796 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:53,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:54,008 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:56,365 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:56,677 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:56,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:59,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:59,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:00,801 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:01,370 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:02,224 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:02,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:04,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:04,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:06,805 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:06,976 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:07,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:24:07,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:24:07,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:07,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:10,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:10,084 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:10,807 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:12,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:12,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:12,658 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:12,711 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:24:12,712 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:24:12,712 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:24:12,713 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:24:12,808 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:24:13,809 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:14,810 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:15,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:15,317 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:16,811 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:17,687 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:17,976 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:17,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:19,960 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:24:22,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:24:22,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:24:23,400 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:25,946 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:24:27,206 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:27,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:28,401 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:29,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:29,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:32,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:32,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:32,823 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:33,823 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:35,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:35,275 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:37,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:24:37,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:24:37,860 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:37,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:39,405 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:40,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:40,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:42,830 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:43,126 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:43,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:44,996 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:45,815 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:45,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:48,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:48,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:48,834 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:49,962 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:24:50,587 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:50,647 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:24:50,648 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:24:50,648 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:24:50,648 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:24:50,836 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:24:51,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:51,288 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:51,837 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:52,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:24:52,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:24:52,838 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:53,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:53,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:54,839 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:56,545 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:56,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:56,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:56,841 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:59,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:59,228 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:01,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:01,874 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:01,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:04,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:04,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:06,558 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:07,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:25:07,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:25:08,855 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:12,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:12,375 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:25:13,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:13,694 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:16,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:16,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:17,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:18,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:19,012 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:19,964 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:25:20,864 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:21,746 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:21,764 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:22,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:25:22,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:25:22,357 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:24,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:24,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:26,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:25:26,258 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:25:26,258 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:25:26,259 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:25:26,867 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:26,868 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:25:27,061 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:27,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:27,868 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:28,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:29,870 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:29,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:32,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:32,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:32,872 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:33,578 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:34,874 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:35,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:35,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:36,880 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:37,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:25:37,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:25:37,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:37,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:39,403 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:40,444 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:40,459 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:43,031 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:43,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:44,404 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:45,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:45,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:46,889 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:48,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:48,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:49,422 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:49,966 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:25:51,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:51,201 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:52,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:25:52,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:25:55,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:59,107 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:25:59,131 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:59,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:59,562 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:25:59,563 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:25:59,563 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:25:59,563 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:25:59,899 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:26:00,564 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:00,900 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:01,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:01,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:01,901 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:04,391 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:04,420 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:05,565 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:07,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:07,138 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:07,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:26:07,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:26:08,907 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:09,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:09,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:11,398 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:12,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:12,598 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:12,909 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:15,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:15,126 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:16,912 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:16,914 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:17,747 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:17,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:18,914 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:19,232 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:26:19,234 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:26:19,234 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:26:19,234 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:26:19,914 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:19,914 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:26:19,969 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:26:20,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:20,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:20,915 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:21,970 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:22,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:26:22,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:26:23,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:23,105 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:25,753 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:25,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:27,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:28,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:28,429 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:31,134 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:31,162 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:32,354 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:33,811 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:33,836 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:36,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:36,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:37,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:26:37,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:26:37,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:40,928 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:42,763 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:44,560 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:26:45,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:45,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:48,101 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:48,259 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:48,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:48,934 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:49,972 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:26:50,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:50,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:52,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:26:52,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:26:53,517 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:53,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:53,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:54,939 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:56,282 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:56,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:58,531 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:58,890 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:58,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:58,942 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:59,401 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:26:59,402 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:26:59,403 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:26:59,403 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:26:59,942 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:27:00,943 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:01,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:01,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:01,944 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:02,945 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:04,081 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:04,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:04,433 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:06,772 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:06,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:07,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:27:07,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:27:09,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:09,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:10,402 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:11,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:11,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:14,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:14,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:15,402 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:17,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:17,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:18,957 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:19,958 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:19,975 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:27:19,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:20,976 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:22,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:27:22,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:27:22,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:22,680 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:26,797 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:28,965 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:30,657 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:27:30,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:30,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:31,798 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:32,967 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:33,330 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:33,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:35,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:35,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:36,366 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:27:36,368 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:27:36,368 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:27:36,368 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:27:36,971 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:36,971 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:27:37,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:37,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:27:37,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:27:37,972 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:38,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:38,562 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:40,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:41,089 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:41,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:42,514 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:42,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:43,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:43,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:46,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:46,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:47,515 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:49,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:49,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:49,977 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:27:51,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:51,709 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:52,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:27:52,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:27:53,349 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:54,672 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:54,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:56,986 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:57,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:57,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:58,908 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:00,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:00,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:02,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:02,772 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:03,908 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:05,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:05,515 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:06,993 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:07,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:28:07,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:28:08,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:08,131 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:09,396 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:10,712 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:28:10,713 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:28:10,714 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:28:10,714 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:28:10,996 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:28:11,998 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:12,998 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:14,715 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:15,926 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:28:15,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:15,994 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:17,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:18,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:18,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:19,004 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:19,981 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:28:19,982 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:21,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:21,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:22,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:28:22,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:28:23,007 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:23,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:23,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:25,404 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:26,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:26,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:29,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:29,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:30,455 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:31,013 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:31,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:31,757 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:34,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:34,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:35,456 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:36,995 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:37,015 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:37,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:28:37,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:28:39,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:39,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:41,329 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:41,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:28:41,681 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:28:41,681 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:28:41,682 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:28:42,020 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:28:42,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:42,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:43,021 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:44,023 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:45,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:45,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:46,683 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:47,773 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:47,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:49,026 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:49,982 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:28:50,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:50,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:51,983 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:52,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:28:52,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:28:53,161 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:53,186 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:55,030 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:57,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:59,034 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:01,035 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:01,637 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:29:01,637 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:29:01,638 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:29:01,638 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:29:01,656 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:29:02,036 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:29:02,639 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:02,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:02,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:03,037 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:04,038 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:05,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:05,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:07,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:29:07,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:29:07,700 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:07,719 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:08,369 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:10,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:10,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:13,046 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:13,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:13,370 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:15,663 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:15,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:18,371 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:18,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:18,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:19,989 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:29:21,053 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:21,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:21,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:22,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:29:22,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:29:23,404 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:23,965 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:23,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:26,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:26,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:29,020 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:29,059 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:29,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:29,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:31,995 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:32,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:34,400 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:34,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:34,697 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:35,068 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:37,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:29:37,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:29:37,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:37,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:39,071 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:39,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:39,945 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:39,997 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:40,050 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:29:40,052 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:29:40,052 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:29:40,052 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:29:40,072 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:29:41,073 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:42,074 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:43,075 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:45,055 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:47,570 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:29:47,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:47,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:49,992 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:29:50,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:50,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:50,994 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:52,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:29:52,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:29:53,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:53,432 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:56,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:56,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:56,399 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:58,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:58,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:59,089 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:01,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:01,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:02,216 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:04,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:04,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:06,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:06,838 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:07,097 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:07,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:30:07,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:30:07,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:09,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:09,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:11,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:11,975 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:12,575 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:13,102 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:14,586 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:14,613 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:15,624 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:30:15,625 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:30:15,625 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:30:15,626 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:30:16,105 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:30:17,106 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:17,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:17,283 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:17,627 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:18,107 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:19,108 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:19,873 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:19,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:19,994 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:30:22,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:30:22,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:30:22,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:22,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:23,112 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:23,350 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:25,394 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:25,416 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:28,351 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:33,121 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:33,804 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:30:33,834 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:33,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:34,008 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:36,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:36,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:37,178 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:30:37,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:30:39,214 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:39,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:39,337 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:41,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:41,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:43,129 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:44,014 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:44,046 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:45,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:46,955 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:46,977 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:47,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:30:47,697 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:30:47,697 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:30:47,698 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:30:48,135 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:30:49,135 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:49,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:49,750 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:49,997 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:30:50,137 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:50,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:52,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:30:52,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:30:52,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:52,576 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:53,139 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:55,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:55,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:56,533 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:57,143 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:57,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:57,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:59,145 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:00,491 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:00,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:02,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:03,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:03,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:05,774 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:05,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:07,134 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:07,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:31:07,177 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:31:08,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:08,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:09,154 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:10,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:11,031 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:13,029 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:18,030 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:19,078 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:31:19,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:19,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:19,999 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:31:21,050 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:31:21,051 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:31:21,051 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:31:21,051 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:31:21,164 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:31:21,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:21,762 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:22,165 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:22,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:31:22,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:31:23,166 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:23,385 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:24,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:24,451 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:27,181 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:27,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:28,855 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:29,171 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:29,830 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:29,850 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:32,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:32,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:34,493 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:34,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:34,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:35,176 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:37,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:31:37,177 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:31:37,206 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:37,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:39,468 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:39,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:39,838 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:41,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:41,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:42,183 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:43,184 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:44,266 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:44,287 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:45,012 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:46,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:46,713 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:49,082 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:49,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:50,001 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:31:51,002 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:51,598 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:51,611 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:52,178 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:31:52,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:31:54,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:54,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:56,396 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:01,397 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:01,454 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:32:02,603 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:02,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:05,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:05,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:06,398 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:07,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:32:07,177 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:32:07,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:07,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:09,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:09,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:11,207 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:12,016 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:12,159 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:12,176 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:13,209 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:14,525 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:14,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:15,126 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 22:32:15,126 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 22:32:15,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:32:15,191 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:32:15,193 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:32:15,210 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:15,211 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav +2024-05-13 22:32:15,211 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav +2024-05-13 22:32:15,211 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json +2024-05-13 22:32:15,211 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 22:32:15,822 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/rarkjo8c-media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json +2024-05-13 22:32:16,060 INFO wandb-upload_1:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmpb6vxw377 +2024-05-13 22:32:16,490 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/o45hsqba-media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav +2024-05-13 22:32:16,498 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/s563f1vp-media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav +2024-05-13 22:32:16,564 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/2u1kfney-media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav +2024-05-13 22:32:16,577 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/2ommzfqv-media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav +2024-05-13 22:32:16,594 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/buiei4kw-media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav +2024-05-13 22:32:16,600 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/0nluwo0n-media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav +2024-05-13 22:32:16,626 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/19s5d7id-media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav +2024-05-13 22:32:16,673 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep762 - {'id': 'QXJ0aWZhY3Q6ODM0ODY2MDEy', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI5MjU4Ng==', 'latestArtifact': None}} +2024-05-13 22:32:16,673 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,674 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json with policy now +2024-05-13 22:32:16,674 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:32:16,674 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:32:16,675 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:32:16,675 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,675 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav with policy now +2024-05-13 22:32:16,675 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,675 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav with policy now +2024-05-13 22:32:16,675 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,675 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav with policy now +2024-05-13 22:32:16,676 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,676 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav with policy now +2024-05-13 22:32:16,676 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,676 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav with policy now +2024-05-13 22:32:16,676 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,676 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav with policy now +2024-05-13 22:32:16,676 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,676 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav with policy now +2024-05-13 22:32:17,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:17,075 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:17,105 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/4b938q0k-media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json +2024-05-13 22:32:17,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:32:17,673 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/chepain9-media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav +2024-05-13 22:32:17,677 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:17,759 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/r8eup6fz-media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav +2024-05-13 22:32:17,765 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/eosarlqk-media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav +2024-05-13 22:32:17,798 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/dgytxsr7-media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav +2024-05-13 22:32:17,812 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/bzf499sl-media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav +2024-05-13 22:32:17,815 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/zsi7xr2u-media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav +2024-05-13 22:32:17,816 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/hc5iidyo-media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav +2024-05-13 22:32:18,213 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:19,214 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:19,730 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:19,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:20,004 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:32:21,217 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:22,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:32:22,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:32:22,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:22,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:23,218 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:23,378 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:23,842 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:32:23,844 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:32:23,844 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:32:23,844 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:32:24,219 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:32:24,943 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:24,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:25,220 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:26,221 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:27,709 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:27,736 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:28,845 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:30,367 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:30,394 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:33,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:33,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:33,846 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:35,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:35,982 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:37,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:32:37,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:32:38,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:38,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:39,314 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:41,234 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:44,882 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:45,237 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:46,847 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:32:46,889 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:46,901 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:47,239 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:49,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:49,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:49,978 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:50,006 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:32:52,178 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:32:52,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:32:52,571 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:52,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:55,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:55,358 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:55,366 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:58,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:58,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:00,367 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:00,720 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:00,755 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:03,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:03,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:05,368 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:06,061 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:06,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:07,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:33:07,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:33:07,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:08,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:08,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:10,758 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:10,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:11,338 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:13,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:13,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:13,375 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:33:13,376 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:33:13,376 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:33:13,376 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:33:14,261 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:14,262 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:33:15,262 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:15,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:15,470 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:16,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:17,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:17,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:20,008 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:33:20,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:20,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:21,267 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:22,009 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:22,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:33:22,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:33:22,873 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:22,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:23,268 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:27,272 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:27,686 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:29,274 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:31,195 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:33:31,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:31,252 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:32,687 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:33,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:33,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:36,178 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:36,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:37,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:33:37,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:33:38,394 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:38,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:38,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:41,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:41,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:43,395 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:43,873 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:33:43,875 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:33:43,875 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:33:43,875 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:33:43,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:43,930 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:44,289 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:33:45,290 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:46,291 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:46,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:46,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:48,876 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:49,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:49,608 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:50,011 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:33:52,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:33:52,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:33:52,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:52,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:54,833 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:55,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:55,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:55,299 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:57,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:57,679 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:00,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:00,317 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:00,454 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:01,304 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:02,951 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:02,982 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:05,307 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:05,508 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:05,838 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:05,862 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:07,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:34:07,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:34:07,309 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:07,751 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:34:07,753 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:34:07,753 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:34:07,753 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:34:08,310 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:34:08,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:08,697 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:09,311 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:10,312 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:10,754 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:15,755 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:17,588 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:34:17,618 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:17,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:20,013 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:34:20,590 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:20,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:21,014 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:21,320 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:22,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:34:22,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:34:23,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:23,570 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:26,370 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:26,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:26,485 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:29,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:29,374 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:31,524 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:32,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:32,252 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:33,329 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:35,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:35,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:36,529 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:37,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:34:37,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:34:38,174 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:38,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:39,333 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:41,177 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:41,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:42,522 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:43,336 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:43,977 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:44,000 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:44,834 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:34:44,835 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:34:44,835 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:34:44,835 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:34:45,338 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:34:46,338 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:46,951 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:46,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:47,339 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:48,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:49,341 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:49,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:49,487 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:50,018 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:34:52,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:34:52,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:34:52,434 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:52,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:53,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:55,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:55,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:57,348 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:57,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:57,875 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:59,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:04,141 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:06,179 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:35:06,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:06,809 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:07,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:35:07,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:35:09,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:09,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:09,606 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:11,360 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:12,064 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:12,088 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:14,606 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:15,072 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:15,093 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:17,365 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:18,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:18,037 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:20,021 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:35:20,023 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:20,948 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:20,968 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:21,521 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:35:21,522 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:35:21,522 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:35:21,522 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:35:22,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:35:22,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:35:22,369 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:35:23,370 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:23,881 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:23,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:24,371 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:25,923 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:26,794 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:26,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:27,373 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:29,375 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:29,684 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:29,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:31,354 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:31,377 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:32,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:32,778 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:35,765 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:35,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:36,355 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:37,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:35:37,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:35:38,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:38,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:41,372 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:41,776 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:41,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:44,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:44,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:46,373 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:47,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:47,704 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:49,392 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:50,024 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:35:52,025 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:52,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:35:52,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:35:55,167 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:35:55,168 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:35:55,168 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:35:55,169 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:35:55,397 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:35:56,398 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:56,451 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:35:56,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:57,169 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:57,399 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:57,833 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:00,776 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:00,797 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:02,496 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:03,405 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:03,719 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:03,736 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:06,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:06,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:07,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:36:07,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:36:07,408 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:08,396 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:09,410 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:09,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:09,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:11,412 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:12,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:12,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:14,190 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:15,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:15,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:18,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:18,468 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:19,191 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:20,026 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:36:21,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:21,437 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:22,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:36:22,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:36:24,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:24,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:24,394 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:25,871 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:36:25,871 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:36:25,872 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:36:25,872 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:36:26,425 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:36:27,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:27,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:27,425 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:28,426 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:29,873 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:30,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:30,248 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:33,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:33,209 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:34,874 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:36,158 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:36,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:37,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:36:37,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:36:37,435 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:39,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:39,129 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:40,340 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:43,441 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:46,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:47,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:48,061 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:36:48,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:49,444 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:49,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:50,029 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:36:50,406 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:36:50,407 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:36:50,408 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:36:50,408 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:36:50,447 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:36:51,409 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:52,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:36:52,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:36:52,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:52,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:52,449 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:53,450 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:55,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:55,416 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:57,388 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:58,374 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:58,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:01,327 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:01,358 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:02,390 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:04,392 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:04,409 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:05,460 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:07,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:37:07,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:37:07,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:07,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:08,381 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:10,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:10,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:13,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:13,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:13,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:16,271 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:16,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:17,472 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:18,814 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:19,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:19,311 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:20,030 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:37:22,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:37:22,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:37:22,339 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:22,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:23,479 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:24,356 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:25,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:25,317 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:27,482 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:28,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:28,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:29,102 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:37:29,103 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:37:29,103 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:37:29,104 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:37:29,484 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:37:30,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:30,485 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:31,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:31,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:31,486 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:33,488 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:35,430 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:37,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:37:37,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:37:39,991 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:37:40,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:40,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:41,406 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:43,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:43,030 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:43,498 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:46,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:46,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:46,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:49,067 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:49,088 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:50,033 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:37:52,034 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:52,150 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:52,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:52,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:37:52,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:37:55,260 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:55,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:55,509 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:57,730 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:58,193 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:58,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:01,198 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:01,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:03,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:03,517 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:04,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:04,174 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:05,241 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:38:05,242 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:38:05,242 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:38:05,243 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:38:05,519 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:38:06,520 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:07,178 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:07,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:38:07,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:38:07,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:07,521 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:08,794 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:10,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:10,193 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:11,525 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:13,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:13,129 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:13,526 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:14,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:15,528 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:16,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:16,077 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:19,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:19,050 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:19,383 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:20,035 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:38:22,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:22,082 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:22,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:38:22,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:38:25,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:30,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:30,896 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:38:30,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:30,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:33,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:33,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:35,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:35,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:36,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:36,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:37,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:38:37,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:38:39,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:39,847 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:41,403 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:41,954 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:38:41,955 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:38:41,956 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:38:41,956 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:38:42,551 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:38:42,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:42,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:43,552 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:44,553 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:45,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:45,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:46,957 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:48,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:48,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:49,558 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:50,038 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:38:51,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:51,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:52,100 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:52,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:38:52,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:38:53,561 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:54,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:54,855 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:55,563 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:57,187 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:57,564 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:57,811 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:57,833 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:00,653 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:00,681 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:02,190 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:03,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:03,624 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:06,701 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:06,713 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:07,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:39:07,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:39:07,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:09,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:09,704 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:12,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:12,671 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:12,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:12,834 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:39:12,835 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:39:12,835 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:39:12,835 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:39:13,579 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:39:14,579 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:15,580 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:17,836 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:20,040 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:39:21,543 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:39:21,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:21,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:22,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:39:22,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:39:23,333 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:23,587 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:24,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:24,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:27,417 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:27,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:28,420 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:29,592 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:30,346 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:30,367 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:33,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:33,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:33,458 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:33,595 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:36,300 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:36,312 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:36,582 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:39:36,583 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:39:36,583 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:39:36,583 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:39:36,598 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:39:37,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:39:37,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:39:37,599 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:38,600 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:39,233 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:39,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:39,356 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:39,600 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:42,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:42,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:44,357 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:45,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:45,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:48,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:48,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:49,358 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:50,042 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:39:51,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:51,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:51,611 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:52,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:39:52,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:39:54,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:54,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:54,372 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:57,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:57,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:59,374 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:00,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:00,267 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:03,328 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:03,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:03,622 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:04,880 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:07,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:40:07,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:40:10,374 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:40:10,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:10,444 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:10,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:11,629 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:13,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:13,443 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:15,632 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:15,860 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:15,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:16,394 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:16,821 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:40:16,822 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:40:16,823 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:40:16,823 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:40:17,634 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:17,634 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:40:18,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:18,879 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:19,636 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:20,045 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:40:21,637 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:21,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:21,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:22,046 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:22,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:40:22,184 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:40:24,827 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:24,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:27,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:27,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:27,772 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:30,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:30,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:31,646 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:32,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:33,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:33,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:36,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:36,665 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:37,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:40:37,184 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:40:38,314 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:39,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:39,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:41,654 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:42,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:42,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:43,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:45,653 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:45,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:48,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:48,722 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:49,096 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:49,660 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:50,047 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:40:51,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:51,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:51,982 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:40:51,983 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:40:51,983 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:40:51,983 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:40:52,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:40:52,184 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:40:52,663 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:40:53,664 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:54,372 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:57,668 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:59,670 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:59,766 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:00,243 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:41:00,282 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:00,293 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:01,671 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:03,158 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:03,172 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:04,768 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:06,150 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:06,174 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:07,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:41:07,184 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:41:09,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:09,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:10,404 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:12,190 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:12,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:15,229 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:15,248 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:15,406 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:18,219 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:18,240 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:20,049 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:41:21,051 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:21,254 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:21,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:21,686 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:22,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:41:22,184 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:41:24,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:24,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:26,310 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:27,062 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:41:27,063 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:41:27,063 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:41:27,063 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:41:27,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:27,219 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:27,691 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:27,691 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:41:29,693 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:30,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:30,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:32,064 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:33,122 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:33,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:35,698 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:36,078 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:36,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:37,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:37,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:41:37,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:41:37,700 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:39,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:39,132 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:41,703 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:42,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:42,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:42,993 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:43,705 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:47,994 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:50,052 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:41:51,124 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:41:51,152 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:51,166 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:52,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:41:52,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:41:53,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:53,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:54,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:57,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:57,078 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:58,292 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:41:58,293 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:41:58,293 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:41:58,294 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:41:58,717 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:41:59,294 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:59,718 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:59,975 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:59,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:02,930 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:02,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:04,296 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:05,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:05,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:07,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:42:07,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:42:08,901 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:08,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:09,440 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:09,726 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:11,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:11,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:14,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:14,930 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:15,090 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:15,731 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:17,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:18,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:19,735 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:20,054 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:42:20,254 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:20,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:20,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:21,737 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:22,185 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:42:22,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:42:22,555 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:42:23,237 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:42:23,237 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:42:23,238 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:42:23,738 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:23,739 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:42:23,810 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:23,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:25,741 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:26,239 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:26,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:26,745 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:29,631 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:29,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:31,240 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:32,607 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:32,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:36,691 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:37,185 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:42:37,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:42:37,751 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:41,157 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:42:41,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:41,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:42,326 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:44,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:44,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:47,733 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:47,760 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:47,772 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:48,148 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:50,057 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:42:50,738 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:50,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:52,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:42:52,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:42:53,607 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:53,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:53,814 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:55,766 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:56,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:56,756 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:59,077 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:59,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:59,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:59,768 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:00,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:43:00,387 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:43:00,387 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:43:00,388 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:43:00,769 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:43:01,770 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:02,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:02,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:04,758 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:05,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:05,555 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:05,773 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:07,185 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:43:07,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:43:08,485 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:08,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:10,368 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:11,557 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:11,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:13,778 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:14,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:14,566 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:16,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:17,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:17,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:20,059 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:43:20,555 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:20,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:22,060 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:22,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:43:22,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:43:23,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:23,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:25,787 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:27,970 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:31,791 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:32,001 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:43:32,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:32,417 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:33,645 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:34,235 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:43:34,236 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:43:34,236 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:43:34,237 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:43:34,793 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:43:35,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:35,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:35,794 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:37,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:43:37,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:43:38,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:38,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:39,523 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:39,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:41,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:41,248 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:41,799 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:43,801 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:44,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:44,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:44,818 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:47,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:47,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:49,818 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:50,061 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:43:50,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:50,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:52,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:43:52,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:43:53,300 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:53,333 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:55,400 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:55,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:56,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:58,660 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:58,693 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:00,402 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:01,614 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:01,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:03,819 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:04,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:04,659 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:06,061 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:07,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:44:07,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:44:07,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:07,597 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:09,385 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:44:09,386 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:44:09,386 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:44:09,386 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:44:09,823 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:09,823 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:44:10,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:10,555 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:11,387 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:11,826 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:13,479 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:13,495 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:16,721 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:17,831 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:20,063 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:44:21,835 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:22,021 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:22,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:44:22,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:44:22,199 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:44:22,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:22,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:23,837 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:25,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:25,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:25,838 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:27,038 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:28,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:28,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:31,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:31,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:32,039 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:34,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:34,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:37,041 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:37,144 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:37,162 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:37,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:44:37,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:44:40,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:40,119 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:44:40,120 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:44:40,120 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:44:40,121 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:44:40,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:40,852 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:44:41,853 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:42,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:43,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:43,082 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:46,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:46,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:47,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:49,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:49,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:50,066 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:44:51,862 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:52,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:52,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:52,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:52,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:44:52,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:44:55,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:55,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:57,867 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:58,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:58,052 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:58,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:00,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:01,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:01,870 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:03,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:03,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:03,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:05,467 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:45:05,468 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:45:05,469 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:45:05,469 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:45:05,874 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:05,875 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:45:07,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:45:07,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:45:07,877 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:08,378 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:12,872 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:45:12,903 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:13,380 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:14,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:17,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:17,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:18,560 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:19,888 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:20,069 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:45:20,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:20,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:22,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:45:22,187 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:45:23,328 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:23,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:24,318 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:26,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:26,309 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:29,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:29,267 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:29,897 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:30,232 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:32,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:32,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:35,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:35,242 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:35,729 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:37,187 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:45:37,187 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:45:37,904 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:38,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:38,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:41,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:41,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:41,597 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:41,908 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:43,048 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:45:43,049 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:45:43,050 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:45:43,050 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:45:43,910 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:43,910 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:45:44,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:44,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:45,911 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:47,251 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:47,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:47,354 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:47,913 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:50,070 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:45:50,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:50,298 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:52,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:45:52,187 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:45:52,337 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:52,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:53,331 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:55,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:55,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:57,922 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:59,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:02,191 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:46:03,665 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:03,702 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:04,286 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:06,735 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:06,757 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:07,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:46:07,187 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:46:09,487 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:09,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:09,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:10,934 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:12,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:12,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:14,487 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:15,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:16,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:16,939 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:18,663 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:18,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:20,074 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:46:20,075 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:21,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:21,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:21,944 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:22,688 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:46:22,688 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:46:24,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:24,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:24,946 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:25,239 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:26,753 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:26,765 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:29,495 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:29,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:30,240 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:32,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:32,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:35,240 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:35,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:35,322 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:37,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:46:37,195 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:46:38,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:38,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:40,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:40,424 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:40,687 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:43,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:43,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:45,688 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:50,075 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:46:50,969 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:51,018 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:51,856 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:46:51,890 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:51,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:52,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:46:52,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:46:52,971 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:54,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:54,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:54,972 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:55,450 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 22:46:55,450 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 22:46:55,456 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:46:55,524 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:46:55,526 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_4871014c416be45744ef.wav +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_819325490f684d367b39.wav +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav +2024-05-13 22:46:55,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json +2024-05-13 22:46:55,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav +2024-05-13 22:46:55,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 22:46:55,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 22:46:56,405 INFO wandb-upload_1:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmp0gy_pukm +2024-05-13 22:46:56,648 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/j9b5d3n0-media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json +2024-05-13 22:46:57,004 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep889 - {'id': 'QXJ0aWZhY3Q6ODM0ODg2Mjc5', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI5ODg3OQ==', 'latestArtifact': None}} +2024-05-13 22:46:57,005 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,005 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json with policy now +2024-05-13 22:46:57,005 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:46:57,005 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:46:57,006 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:57,006 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:46:57,007 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,007 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav with policy now +2024-05-13 22:46:57,007 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,007 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_4871014c416be45744ef.wav with policy now +2024-05-13 22:46:57,007 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,007 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav with policy now +2024-05-13 22:46:57,007 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,009 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav with policy now +2024-05-13 22:46:57,009 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,009 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_819325490f684d367b39.wav with policy now +2024-05-13 22:46:57,009 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,009 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav with policy now +2024-05-13 22:46:57,010 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav with policy now +2024-05-13 22:46:57,093 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/tn4irtcn-media/audio/Speech samples/eval_889_4871014c416be45744ef.wav +2024-05-13 22:46:57,198 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/y0nfu36w-media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav +2024-05-13 22:46:57,219 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/yxbqme64-media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav +2024-05-13 22:46:57,283 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/nxgqgrbc-media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav +2024-05-13 22:46:57,289 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/exuljp42-media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav +2024-05-13 22:46:57,298 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/wij34dh8-media/audio/Speech samples/eval_889_819325490f684d367b39.wav +2024-05-13 22:46:57,307 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/0xe8ygpj-media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav +2024-05-13 22:46:57,433 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/xxf7p9fk-media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json +2024-05-13 22:46:57,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:57,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:57,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:57,976 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:46:58,243 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/x6jox6cv-media/audio/Speech samples/eval_889_4871014c416be45744ef.wav +2024-05-13 22:46:58,350 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/1h918ijy-media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav +2024-05-13 22:46:58,355 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/yb9abl1z-media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav +2024-05-13 22:46:58,433 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/75kqlmkv-media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav +2024-05-13 22:46:58,462 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/km3jn9e7-media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav +2024-05-13 22:46:58,977 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:59,323 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/nu83r2us-media/audio/Speech samples/eval_889_819325490f684d367b39.wav +2024-05-13 22:46:59,464 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/1kwi9m8p-media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav +2024-05-13 22:47:00,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:00,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:01,086 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:47:01,087 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:47:01,088 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:47:01,088 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:47:01,980 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:01,980 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:47:02,088 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:02,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:02,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:02,981 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:05,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:05,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:06,985 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:07,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:47:07,195 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:47:07,350 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:08,102 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:08,128 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:08,987 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:10,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:10,581 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:10,989 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:12,755 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:13,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:13,327 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:16,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:16,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:17,755 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:18,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:18,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:20,078 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:47:21,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:21,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:22,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:47:22,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:47:23,693 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:24,433 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:24,459 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:27,005 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:27,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:27,232 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:29,515 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:29,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:30,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:30,962 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:47:30,964 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:47:30,964 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:47:30,964 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:47:31,009 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:47:32,010 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:33,011 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:35,013 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:35,339 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:37,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:47:37,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:47:39,766 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:47:40,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:40,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:40,693 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:43,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:43,433 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:45,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:46,105 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:46,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:48,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:49,019 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:50,080 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:47:51,082 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:51,576 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:51,589 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:52,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:47:52,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:47:54,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:54,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:56,704 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:56,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:56,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:57,035 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:59,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:59,526 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:01,706 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:02,180 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:02,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:03,042 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:04,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:04,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:07,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:48:07,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:48:07,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:07,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:07,746 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:10,251 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:10,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:11,050 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:12,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:12,620 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:48:12,620 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:48:12,620 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:48:12,621 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:48:13,052 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:48:13,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:13,394 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:14,053 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:15,054 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:15,932 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:15,942 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:17,056 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:18,393 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:18,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:18,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:19,058 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:20,082 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:48:22,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:48:22,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:48:23,693 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:26,364 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:48:26,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:26,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:28,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:28,949 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:28,959 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:31,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:31,610 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:34,064 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:34,156 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:34,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:35,072 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:36,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:36,719 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:37,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:48:37,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:48:39,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:39,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:39,693 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:42,045 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:42,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:43,079 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:44,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:44,711 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:45,287 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:46,876 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:48:46,877 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:48:46,877 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:48:46,878 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:48:47,083 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:48:47,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:47,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:48,084 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:49,085 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:49,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:49,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:50,085 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:48:50,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:52,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:48:52,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:48:52,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:52,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:53,091 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:55,093 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:55,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:55,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:56,544 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:57,095 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:57,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:57,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:00,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:00,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:01,546 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:02,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:03,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:06,546 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:07,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:49:07,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:49:09,105 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:10,827 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:49:10,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:10,870 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:12,290 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:13,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:13,465 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:16,031 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:16,044 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:17,291 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:18,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:18,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:19,412 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:49:19,413 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:49:19,413 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:49:19,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:49:20,088 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:49:20,114 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:20,114 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:49:21,115 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:21,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:21,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:22,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:49:22,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:49:22,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:23,944 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:23,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:27,120 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:27,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:27,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:28,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:30,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:30,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:33,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:33,019 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:33,125 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:33,954 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:35,127 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:35,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:35,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:37,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:49:37,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:49:38,105 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:38,118 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:39,131 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:39,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:40,735 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:40,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:43,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:43,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:44,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:45,935 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:49:45,936 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:49:45,936 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:49:45,936 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:49:45,962 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:45,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:46,137 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:49:47,138 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:48,139 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:48,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:48,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:49,937 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:50,091 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:49:52,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:49:52,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:49:55,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:56,767 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:49:56,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:56,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:59,148 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:59,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:59,370 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:01,518 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:01,937 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:01,948 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:04,737 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:04,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:07,029 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:07,156 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:07,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:50:07,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:50:07,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:07,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:10,374 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:10,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:12,376 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:12,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:12,945 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:13,161 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:15,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:15,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:17,165 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:17,611 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:17,661 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:50:17,662 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:50:17,662 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:50:17,662 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:50:18,166 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:18,166 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:50:19,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:19,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:19,166 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:20,095 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:50:21,168 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:21,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:21,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:22,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:50:22,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:50:22,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:24,589 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:24,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:27,192 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:27,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:27,695 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:29,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:29,884 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:32,468 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:32,480 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:32,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:35,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:35,030 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:37,197 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:50:37,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:50:38,695 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:41,185 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:43,722 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:43,916 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:50:44,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:44,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:46,972 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:46,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:47,190 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:49,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:49,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:49,571 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:50,095 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:50:52,197 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:50:52,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:50:52,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:52,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:53,195 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:54,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:54,955 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:54,966 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:55,561 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:50:55,562 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:50:55,562 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:50:55,563 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:50:56,199 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:56,199 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:50:57,200 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:57,611 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:57,622 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:59,202 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:00,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:00,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:00,181 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:01,204 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:02,665 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:02,683 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:05,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:05,242 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:05,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:07,197 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:51:07,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:51:08,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:08,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:10,695 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:11,030 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:11,044 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:13,755 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:13,765 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:15,713 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:16,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:16,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:17,221 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:18,990 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:19,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:20,097 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:51:21,099 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:21,680 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:21,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:22,197 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:51:22,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:51:25,230 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:26,630 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:29,469 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:51:29,470 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:51:29,470 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:51:29,471 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:51:29,605 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:51:30,093 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:30,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:30,235 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:30,235 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:51:31,236 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:32,472 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:32,581 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:32,593 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:35,152 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:35,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:35,240 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:37,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:51:37,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:51:37,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:37,695 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:37,889 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:37,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:39,244 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:40,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:40,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:43,028 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:43,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:43,588 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:45,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:45,599 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:48,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:48,274 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:48,588 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:50,100 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:51:50,850 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:50,862 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:52,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:51:52,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:51:53,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:53,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:53,434 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:53,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:56,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:56,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:58,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:58,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:58,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:01,783 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:01,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:02,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:52:02,122 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:52:02,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:52:02,123 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:52:02,265 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:52:03,266 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:04,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:04,267 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:04,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:04,396 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:07,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:07,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:07,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:52:07,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:52:09,229 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:09,272 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:14,358 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:52:14,645 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:15,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:15,215 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:15,278 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:17,280 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:17,957 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:18,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:19,282 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:20,102 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:52:20,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:20,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:20,232 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:22,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:52:22,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:52:22,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:22,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:25,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:25,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:25,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:27,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:27,610 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:28,699 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:52:28,700 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:52:28,700 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:52:28,701 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:52:29,291 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:29,292 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:52:30,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:30,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:30,292 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:30,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:32,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:32,319 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:34,638 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:34,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:35,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:36,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:36,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:37,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:52:37,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:52:39,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:39,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:41,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:41,620 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:41,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:43,304 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:43,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:43,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:46,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:46,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:46,622 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:48,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:48,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:49,309 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:50,105 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:52:52,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:52,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:52:52,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:52:55,316 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:55,942 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:52:55,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:55,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:57,215 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:58,311 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:58,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:59,319 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:00,488 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:53:00,489 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:53:00,489 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:53:00,489 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:53:00,571 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:00,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:01,321 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:01,321 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:53:02,322 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:02,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:02,757 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:02,771 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:03,323 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:05,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:05,068 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:07,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:53:07,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:53:07,356 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:07,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:07,369 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:09,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:09,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:12,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:12,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:12,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:14,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:14,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:17,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:17,465 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:17,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:20,108 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:53:20,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:20,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:21,339 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:22,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:53:22,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:53:22,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:22,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:23,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:25,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:25,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:28,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:28,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:28,801 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:29,346 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:30,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:30,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:33,941 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:35,351 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:36,873 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:53:36,874 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:53:36,874 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:53:36,874 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:53:37,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:53:37,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:53:37,354 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:53:38,354 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:39,356 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:39,534 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:53:39,575 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:40,149 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:40,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:41,357 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:42,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:42,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:43,359 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:45,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:45,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:45,571 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:47,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:47,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:50,111 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:53:50,941 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:50,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:51,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:52,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:53:52,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:53:53,514 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:53,527 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:56,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:56,232 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:56,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:58,797 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:58,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:59,375 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:01,356 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:01,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:01,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:04,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:04,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:06,941 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:06,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:07,087 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:07,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:54:07,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:54:07,388 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:09,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:09,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:11,629 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:54:11,630 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:54:11,630 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:54:11,631 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:54:12,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:12,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:12,393 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:54:12,631 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:13,394 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:14,395 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:15,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:15,037 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:17,344 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:17,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:17,398 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:18,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:19,399 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:20,112 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:54:21,401 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:22,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:54:22,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:54:23,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:25,580 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:54:25,610 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:25,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:28,358 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:28,369 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:28,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:30,891 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:30,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:33,099 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:33,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:34,337 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:35,414 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:35,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:35,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:37,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:54:37,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:54:38,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:38,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:39,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:41,069 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:41,081 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:43,688 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:43,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:44,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:54:44,143 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:54:44,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:54:44,144 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:54:44,422 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:54:45,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:45,423 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:46,271 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:46,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:46,424 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:48,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:48,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:50,115 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:54:50,253 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:51,427 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:51,845 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:51,859 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:52,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:54:52,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:54:54,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:54,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:55,431 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:55,501 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:57,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:57,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:59,434 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:59,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:59,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:00,837 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:01,436 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:02,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:02,820 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:05,837 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:07,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:55:07,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:55:09,624 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:55:09,625 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:55:09,625 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:55:09,626 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:55:10,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:55:11,255 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:55:11,447 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:11,626 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:11,949 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:11,964 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:12,448 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:14,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:14,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:16,627 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:17,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:17,597 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:20,117 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:55:20,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:20,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:22,119 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:22,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:55:22,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:55:22,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:22,808 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:23,458 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:25,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:25,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:27,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:28,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:28,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:30,823 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:30,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:31,466 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:32,881 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:33,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:33,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:35,470 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:36,072 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:36,515 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:37,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:55:37,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:55:38,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:39,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:39,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:39,474 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:39,981 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:55:39,981 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:55:39,982 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:55:39,982 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:55:40,475 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:55:41,476 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:41,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:41,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:42,477 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:43,478 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:43,884 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:44,136 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:44,149 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:46,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:46,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:48,885 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:49,388 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:49,829 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:50,120 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:55:52,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:55:52,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:55:54,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:57,279 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:55:57,666 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:57,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:00,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:00,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:00,678 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:01,494 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:02,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:02,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:05,679 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:05,827 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:05,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:07,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:56:07,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:56:08,480 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:08,491 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:09,502 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:10,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:11,122 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:11,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:13,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:13,774 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:15,507 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:15,777 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:16,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:16,736 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:16,805 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:56:16,805 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:56:16,806 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:56:16,806 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:56:17,509 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:56:18,510 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:19,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:19,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:19,511 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:20,122 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:56:21,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:21,513 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:21,780 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:21,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:22,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:56:22,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:56:23,515 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:24,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:24,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:26,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:26,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:26,990 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:29,964 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:29,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:31,700 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:32,534 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:32,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:35,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:35,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:36,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:37,200 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:56:37,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:56:39,531 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:41,949 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:43,475 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:56:44,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:44,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:46,950 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:47,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:47,152 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:49,540 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:49,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:49,730 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:50,125 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:56:52,126 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:52,200 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:56:52,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:56:52,711 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:52,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:53,060 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:56:53,062 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:56:53,062 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:56:53,062 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:56:53,543 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:56:54,544 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:55,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:55,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:55,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:57,169 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:58,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:58,031 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:59,548 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:00,599 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:00,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:01,551 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:02,230 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:03,553 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:03,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:03,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:06,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:06,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:07,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:57:07,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:57:07,367 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:08,820 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:08,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:11,463 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:11,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:12,699 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:14,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:14,552 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:17,121 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:17,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:17,567 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:18,524 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:19,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:19,701 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:20,126 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:57:22,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:57:22,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:57:22,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:22,231 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:23,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:27,070 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:57:27,071 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:57:27,071 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:57:27,072 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:57:27,576 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:57:28,577 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:29,073 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:29,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:31,255 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:57:31,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:31,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:34,249 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:34,367 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:34,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:35,583 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:37,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:57:37,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:57:37,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:37,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:39,587 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:39,822 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:40,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:40,064 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:42,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:42,670 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:43,591 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:45,264 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:45,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:45,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:45,593 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:47,896 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:47,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:50,130 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:57:50,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:50,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:51,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:52,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:57:52,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:57:52,239 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:57:52,358 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:57:52,359 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:57:52,359 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:57:52,598 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:57:53,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:53,581 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:53,600 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:54,601 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:56,175 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:56,190 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:56,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:58,739 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:58,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:01,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:01,420 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:01,703 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:04,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:04,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:05,610 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:06,812 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:07,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:07,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:07,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:58:07,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:58:09,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:09,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:11,616 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:12,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:17,622 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:17,915 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:58:18,333 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:19,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:19,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:20,131 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:58:21,477 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:58:21,478 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:58:21,478 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:58:21,478 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:58:21,625 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:21,625 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:58:22,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:58:22,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:58:22,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:22,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:23,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:23,626 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:24,627 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:25,443 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:25,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:25,628 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:28,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:28,068 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:28,337 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:30,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:30,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:33,338 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:33,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:33,641 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:36,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:36,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:37,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:58:37,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:58:38,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:38,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:38,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:41,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:41,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:43,644 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:44,028 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:44,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:44,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:46,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:46,633 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:49,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:49,291 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:49,302 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:50,135 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:58:51,650 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:51,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:51,879 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:52,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:58:52,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:58:54,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:54,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:54,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:57,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:57,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:57,654 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:00,399 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:00,521 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:59:00,522 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:59:00,522 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:59:00,522 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:59:00,657 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:59:01,658 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:02,660 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:04,937 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:59:04,959 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:05,533 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:05,664 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:06,393 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:07,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:59:07,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:59:07,667 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:08,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:08,680 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:11,391 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:11,402 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:11,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:14,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:14,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:16,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:16,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:16,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:19,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:19,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:20,138 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:59:21,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:21,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:22,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:59:22,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:59:22,364 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:23,680 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:24,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:24,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:27,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:27,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:27,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:29,738 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:29,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:31,689 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:32,309 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:32,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:33,434 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:34,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:34,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:36,282 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:59:36,283 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:59:36,283 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:59:36,283 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:59:36,693 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:59:37,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:59:37,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:59:37,581 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:37,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:37,693 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:38,694 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:38,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:40,128 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:40,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:41,697 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:42,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:42,688 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:44,496 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:45,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:47,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:49,765 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:50,139 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:59:51,610 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:59:51,642 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:51,651 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:52,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:59:52,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:59:54,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:54,291 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:55,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:56,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:56,860 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:59,514 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:59,524 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:01,258 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:01,715 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:02,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:02,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:04,771 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:04,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:06,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:07,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:00:07,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:00:07,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:07,366 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:09,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 23:00:09,323 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 23:00:09,323 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 23:00:09,323 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:00:09,722 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:00:10,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:10,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:10,722 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:11,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:11,723 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:12,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:12,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:15,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:15,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:16,390 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:17,729 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:17,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:17,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:20,142 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 23:00:20,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:20,425 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:21,732 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:22,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:22,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:00:22,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:00:23,045 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:23,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:25,627 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:25,638 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:25,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:27,229 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:27,738 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:28,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:28,302 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:32,230 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:35,829 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 23:00:35,830 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 23:00:35,831 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 23:00:35,831 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:00:36,073 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 23:00:36,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:36,747 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:00:36,997 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:37,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:00:37,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:00:37,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:37,748 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:38,749 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:39,930 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:39,945 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:42,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:42,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:42,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:45,149 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:45,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:47,704 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:47,713 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:47,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:50,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:50,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:50,145 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 23:00:51,760 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:52,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:00:52,203 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:00:52,403 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:52,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:53,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:54,738 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:54,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:56,764 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:56,975 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:56,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:57,765 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:58,832 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:59,193 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:59,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:01,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:01,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:01,769 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:03,771 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:03,905 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:03,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:04,740 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:06,292 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:06,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:07,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:01:07,203 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:01:08,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:08,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:10,703 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:11,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:11,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:15,704 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:18,749 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 23:01:19,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:19,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:20,147 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 23:01:21,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:21,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:21,735 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:22,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:01:22,203 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:01:24,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:24,161 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:26,557 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:26,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:26,703 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:28,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:28,910 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:29,794 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:31,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:31,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:31,796 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:31,984 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 23:01:31,985 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 23:01:31,991 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 23:01:32,050 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 23:01:32,051 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 23:01:32,797 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav +2024-05-13 23:01:32,797 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json +2024-05-13 23:01:32,797 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav +2024-05-13 23:01:32,797 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 23:01:32,875 INFO wandb-upload_3:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmp0dy8l2wz +2024-05-13 23:01:33,304 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/3wximedk-media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json +2024-05-13 23:01:33,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:33,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:33,491 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:33,599 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep1k - {'id': 'QXJ0aWZhY3Q6ODM0OTA1MzIx', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDMwMjM1OQ==', 'latestArtifact': None}} +2024-05-13 23:01:33,599 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,600 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json with policy now +2024-05-13 23:01:33,600 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:33,600 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 23:01:33,600 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 23:01:33,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:01:33,824 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/fmxfuw3j-media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav +2024-05-13 23:01:33,879 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/f9ey195a-media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav +2024-05-13 23:01:33,901 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:01:33,901 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,901 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav with policy now +2024-05-13 23:01:33,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav with policy now +2024-05-13 23:01:33,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav with policy now +2024-05-13 23:01:33,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav with policy now +2024-05-13 23:01:33,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav with policy now +2024-05-13 23:01:33,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,903 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav with policy now +2024-05-13 23:01:33,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,903 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav with policy now +2024-05-13 23:01:33,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: telemetry +2024-05-13 23:01:33,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: exit +2024-05-13 23:01:33,903 INFO SenderThread:1257956 [sender.py:send_exit():585] handling exit code: 0 +2024-05-13 23:01:33,906 INFO SenderThread:1257956 [sender.py:send_exit():587] handling runtime: 7723 +2024-05-13 23:01:33,986 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/0gk25oxx-media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav +2024-05-13 23:01:34,004 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/xm3tzz4p-media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav +2024-05-13 23:01:34,018 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/9mg7firf-media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav +2024-05-13 23:01:34,024 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/we9rf8az-media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav +2024-05-13 23:01:34,025 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/kqnjmta5-media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json +2024-05-13 23:01:34,041 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/tuqrjrmb-media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav +2024-05-13 23:01:34,477 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:01:34,477 INFO SenderThread:1257956 [sender.py:send_exit():593] send defer +2024-05-13 23:01:34,477 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:34,478 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:34,478 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 0 +2024-05-13 23:01:34,478 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:34,478 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 0 +2024-05-13 23:01:34,478 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 1 +2024-05-13 23:01:34,478 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:34,478 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 1 +2024-05-13 23:01:34,479 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:34,479 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 1 +2024-05-13 23:01:34,479 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 2 +2024-05-13 23:01:34,479 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:34,479 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 2 +2024-05-13 23:01:34,479 INFO HandlerThread:1257956 [system_monitor.py:finish():203] Stopping system monitor +2024-05-13 23:01:34,479 DEBUG SystemMonitor:1257956 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-13 23:01:34,480 DEBUG SystemMonitor:1257956 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-13 23:01:34,480 INFO HandlerThread:1257956 [interfaces.py:finish():200] Joined cpu monitor +2024-05-13 23:01:34,482 INFO HandlerThread:1257956 [interfaces.py:finish():200] Joined disk monitor +2024-05-13 23:01:34,799 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:01:34,899 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/n09f170t-media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav +2024-05-13 23:01:35,041 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/h6xp0rko-media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav +2024-05-13 23:01:35,055 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/77pqyzc6-media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav +2024-05-13 23:01:35,066 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ekemvu9m-media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav +2024-05-13 23:01:35,148 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/bt303oft-media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav +2024-05-13 23:01:35,800 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:35,994 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/p0zwlfn5-media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav +2024-05-13 23:01:36,192 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/uaw9etzp-media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav +2024-05-13 23:01:36,801 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:39,418 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 23:01:39,419 INFO HandlerThread:1257956 [interfaces.py:finish():200] Joined gpu monitor +2024-05-13 23:01:39,419 INFO HandlerThread:1257956 [interfaces.py:finish():200] Joined memory monitor +2024-05-13 23:01:39,419 INFO HandlerThread:1257956 [interfaces.py:finish():200] Joined network monitor +2024-05-13 23:01:39,419 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:39,421 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,421 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 2 +2024-05-13 23:01:39,421 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 3 +2024-05-13 23:01:39,421 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 23:01:39,422 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:39,423 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:39,423 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 3 +2024-05-13 23:01:39,423 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:39,424 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 23:01:39,425 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 23:01:39,425 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:01:39,425 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,425 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 3 +2024-05-13 23:01:39,425 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 4 +2024-05-13 23:01:39,426 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:39,426 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 4 +2024-05-13 23:01:39,426 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,426 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 4 +2024-05-13 23:01:39,426 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 5 +2024-05-13 23:01:39,426 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:39,426 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 5 +2024-05-13 23:01:39,429 DEBUG SenderThread:1257956 [sender.py:send():378] send: summary +2024-05-13 23:01:39,429 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:01:39,429 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,429 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 5 +2024-05-13 23:01:39,429 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 6 +2024-05-13 23:01:39,430 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:39,430 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 6 +2024-05-13 23:01:39,430 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,430 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 6 +2024-05-13 23:01:39,432 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:39,493 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: keepalive +2024-05-13 23:01:39,493 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:39,583 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 7 +2024-05-13 23:01:39,583 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:39,584 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:39,584 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 7 +2024-05-13 23:01:39,585 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,585 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 7 +2024-05-13 23:01:39,804 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/config.yaml +2024-05-13 23:01:39,804 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:01:40,494 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:41,278 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 8 +2024-05-13 23:01:41,278 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:41,279 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:41,279 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 8 +2024-05-13 23:01:41,279 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:41,279 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 8 +2024-05-13 23:01:41,279 INFO SenderThread:1257956 [job_builder.py:build():432] Attempting to build job artifact +2024-05-13 23:01:41,280 INFO SenderThread:1257956 [job_builder.py:_get_source_type():565] is repo sourced job +2024-05-13 23:01:41,282 INFO SenderThread:1257956 [job_builder.py:build():541] adding wandb-job metadata file +2024-05-13 23:01:41,289 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 9 +2024-05-13 23:01:41,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:41,289 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 9 +2024-05-13 23:01:41,290 DEBUG SenderThread:1257956 [sender.py:send():378] send: artifact +2024-05-13 23:01:41,494 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:41,806 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:42,002 INFO wandb-upload_0:1257956 [upload_job.py:push():85] Skipped uploading /home/sanchit/.local/share/wandb/artifacts/staging/tmpantujzmu +2024-05-13 23:01:42,278 INFO wandb-upload_6:1257956 [upload_job.py:push():88] Uploaded file /tmp/tmpcxnm05wb/wandb-job.json +2024-05-13 23:01:43,120 INFO SenderThread:1257956 [sender.py:send_artifact():1467] sent artifact job-https___huggingface.co_sanchit-gandhi_parler-tts-mini-v0.1-expresso-concatenated-combined_run_parler_tts_training.py - {'id': 'QXJ0aWZhY3Q6ODM0OTA1NDk0', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDIzMTI3NQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6ODM0NzI5NzMx', 'versionIndex': 0}}} +2024-05-13 23:01:43,120 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:43,120 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 9 +2024-05-13 23:01:43,120 INFO SenderThread:1257956 [dir_watcher.py:finish():358] shutting down directory watcher +2024-05-13 23:01:43,807 INFO SenderThread:1257956 [dir_watcher.py:finish():388] scan: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/conda-environment.yaml conda-environment.yaml +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json wandb-summary.json +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log output.log +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/requirements.txt requirements.txt +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/config.yaml config.yaml +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-metadata.json wandb-metadata.json +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_492597073098578f0605.wav media/audio/Speech samples/eval_381_492597073098578f0605.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_4871014c416be45744ef.wav media/audio/Speech samples/eval_889_4871014c416be45744ef.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_819325490f684d367b39.wav media/audio/Speech samples/eval_889_819325490f684d367b39.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json +2024-05-13 23:01:43,819 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json +2024-05-13 23:01:43,819 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 10 +2024-05-13 23:01:43,819 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:43,819 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:43,819 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 10 +2024-05-13 23:01:43,820 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:43,820 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 10 +2024-05-13 23:01:43,820 INFO SenderThread:1257956 [file_pusher.py:finish():169] shutting down file pusher +2024-05-13 23:01:44,141 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/conda-environment.yaml +2024-05-13 23:01:44,146 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/requirements.txt +2024-05-13 23:01:44,153 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/config.yaml +2024-05-13 23:01:44,161 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:01:44,374 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:44,496 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:44,496 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:44,497 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:44,575 INFO Thread-11 (_thread_body):1257956 [sender.py:transition_state():613] send defer: 11 +2024-05-13 23:01:44,575 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:44,575 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 11 +2024-05-13 23:01:44,576 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:44,576 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 11 +2024-05-13 23:01:44,576 INFO SenderThread:1257956 [file_pusher.py:join():175] waiting for file pusher +2024-05-13 23:01:44,584 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 12 +2024-05-13 23:01:44,584 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:44,584 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 12 +2024-05-13 23:01:44,585 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:44,585 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 12 +2024-05-13 23:01:44,585 INFO SenderThread:1257956 [file_stream.py:finish():601] file stream finish called +2024-05-13 23:01:45,112 INFO SenderThread:1257956 [file_stream.py:finish():605] file stream finish is done +2024-05-13 23:01:45,112 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 13 +2024-05-13 23:01:45,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:45,112 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 13 +2024-05-13 23:01:45,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:45,113 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 13 +2024-05-13 23:01:45,113 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 14 +2024-05-13 23:01:45,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:45,113 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 14 +2024-05-13 23:01:45,114 DEBUG SenderThread:1257956 [sender.py:send():378] send: final +2024-05-13 23:01:45,114 DEBUG SenderThread:1257956 [sender.py:send():378] send: footer +2024-05-13 23:01:45,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:45,114 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 14 +2024-05-13 23:01:45,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:45,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:45,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: internal_messages +2024-05-13 23:01:45,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: server_info +2024-05-13 23:01:45,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: get_summary +2024-05-13 23:01:45,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: sampled_history +2024-05-13 23:01:45,120 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: server_info +2024-05-13 23:01:45,251 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: shutdown +2024-05-13 23:01:45,252 INFO HandlerThread:1257956 [handler.py:finish():882] shutting down handler +2024-05-13 23:01:46,118 INFO WriterThread:1257956 [datastore.py:close():296] close: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/run-qaoje1x9.wandb +2024-05-13 23:01:46,250 INFO SenderThread:1257956 [sender.py:finish():1545] shutting down sender +2024-05-13 23:01:46,250 INFO SenderThread:1257956 [file_pusher.py:finish():169] shutting down file pusher +2024-05-13 23:01:46,250 INFO SenderThread:1257956 [file_pusher.py:join():175] waiting for file pusher diff --git a/wandb/debug.log b/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..eec1410311289077b92c6ea54355d9093f5a6ed4 --- /dev/null +++ b/wandb/debug.log @@ -0,0 +1,35 @@ +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Current SDK version is 0.17.0 +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Configure stats pid to 1257680 +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Loading settings from /home/sanchit/.config/wandb/settings +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Loading settings from /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/settings +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'run_parler_tts_training.py', 'program_abspath': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py', 'program': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py'} +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:_log_setup():520] Logging user logs to /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/logs/debug.log +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:_log_setup():521] Logging internal logs to /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/logs/debug-internal.log +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:init():560] calling init triggers +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:init():567] wandb.init called with sweep_config: {} +config: {} +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:init():610] starting backend +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:init():614] setting up manager +2024-05-13 20:52:49,019 INFO MainThread:1257680 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-13 20:52:49,019 INFO MainThread:1257680 [wandb_init.py:init():622] backend started and connected +2024-05-13 20:52:49,021 INFO MainThread:1257680 [wandb_init.py:init():711] updated telemetry +2024-05-13 20:52:49,024 INFO MainThread:1257680 [wandb_init.py:init():744] communicating run to backend with 90.0 second timeout +2024-05-13 20:52:49,413 INFO MainThread:1257680 [wandb_run.py:_on_init():2396] communicating current version +2024-05-13 20:52:49,474 INFO MainThread:1257680 [wandb_run.py:_on_init():2405] got version response +2024-05-13 20:52:49,475 INFO MainThread:1257680 [wandb_init.py:init():795] starting run threads in backend +2024-05-13 20:52:52,103 INFO MainThread:1257680 [wandb_run.py:_console_start():2374] atexit reg +2024-05-13 20:52:52,103 INFO MainThread:1257680 [wandb_run.py:_redirect():2229] redirect: wrap_raw +2024-05-13 20:52:52,103 INFO MainThread:1257680 [wandb_run.py:_redirect():2294] Wrapping output streams. +2024-05-13 20:52:52,103 INFO MainThread:1257680 [wandb_run.py:_redirect():2319] Redirects installed. +2024-05-13 20:52:52,104 INFO MainThread:1257680 [wandb_init.py:init():838] run started, returning control to user process +2024-05-13 20:52:52,104 INFO MainThread:1257680 [wandb_run.py:_config_callback():1376] config_cb None None {'learning_rate': 8e-05, 'model_name_or_path': 'parler-tts/parler_tts_mini_v0.1', 'num_train_epochs': 8, 'gradient_accumulation_steps': 8, 'per_device_train_batch_size': 16, 'global_batch_size': 16, 'mixed_precision': 'bf16', 'lr_scheduler_type': 'SchedulerType.COSINE', 'warmup_steps': 250, 'freeze_text_encoder': True, 'max_duration_in_seconds': 30.0, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.99, 'temperature': 1.0} +2024-05-13 23:01:32,489 INFO MainThread:1257680 [wandb_run.py:_finish():2103] finishing run sanchit-gandhi/parler-speech/qaoje1x9 +2024-05-13 23:01:32,489 INFO MainThread:1257680 [wandb_run.py:_atexit_cleanup():2343] got exitcode: 0 +2024-05-13 23:01:32,489 INFO MainThread:1257680 [wandb_run.py:_restore():2326] restore +2024-05-13 23:01:32,489 INFO MainThread:1257680 [wandb_run.py:_restore():2332] restore done +2024-05-13 23:01:46,253 INFO MainThread:1257680 [wandb_run.py:_footer_history_summary_info():3994] rendering history +2024-05-13 23:01:46,254 INFO MainThread:1257680 [wandb_run.py:_footer_history_summary_info():4026] rendering summary +2024-05-13 23:01:46,256 INFO MainThread:1257680 [wandb_run.py:_footer_sync_info():3953] logging synced files diff --git a/wandb/run-20240513_204652-m0g0ap7d/files/conda-environment.yaml b/wandb/run-20240513_204652-m0g0ap7d/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ecd1ac96b302b32274bdbd12ac08d3e1d65a777 --- /dev/null +++ b/wandb/run-20240513_204652-m0g0ap7d/files/conda-environment.yaml @@ -0,0 +1,248 @@ +name: venv +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - bzip2=1.0.8=h5eee18b_6 + - ca-certificates=2024.3.11=h06a4308_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_1 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - libuuid=1.41.5=h5eee18b_0 + - ncurses=6.4=h6a678d5_0 + - openssl=3.0.13=h7f8727e_1 + - pip=24.0=py311h06a4308_0 + - python=3.11.9=h955ad1f_0 + - readline=8.2=h5eee18b_0 + - setuptools=69.5.1=py311h06a4308_0 + - sqlite=3.45.3=h5eee18b_0 + - tk=8.6.14=h39e8969_0 + - wheel=0.43.0=py311h06a4308_0 + - xz=5.4.6=h5eee18b_1 + - zlib=1.2.13=h5eee18b_1 + - pip: + - absl-py==2.1.0 + - accelerate==0.30.0 + - aiohttp==3.9.5 + - aiosignal==1.3.1 + - aniso8601==9.0.1 + - annotated-types==0.6.0 + - anyio==4.3.0 + - argbind==0.3.7 + - argon2-cffi==23.1.0 + - argon2-cffi-bindings==21.2.0 + - arrow==1.3.0 + - asttokens==2.4.1 + - async-lru==2.0.4 + - attrs==23.2.0 + - audioread==3.0.1 + - babel==2.15.0 + - beautifulsoup4==4.12.3 + - bidict==0.23.1 + - bitsandbytes==0.43.1 + - bleach==6.1.0 + - certifi==2024.2.2 + - cffi==1.16.0 + - charset-normalizer==3.3.2 + - click==8.1.7 + - coloredlogs==14.0 + - comm==0.2.2 + - contourpy==1.2.1 + - cycler==0.12.1 + - datasets==2.19.1 + - debugpy==1.8.1 + - decorator==5.1.1 + - defusedxml==0.7.1 + - descript-audio-codec==1.0.0 + - descript-audiotools==0.7.2 + - dill==0.3.8 + - dnspython==2.3.0 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - editdistance==0.8.1 + - einops==0.8.0 + - et-xmlfile==1.1.0 + - evaluate==0.4.2 + - eventlet==0.36.1 + - executing==2.0.1 + - fastjsonschema==2.19.1 + - ffmpy==0.3.2 + - filelock==3.14.0 + - fire==0.6.0 + - flask==2.2.5 + - flask-cors==4.0.1 + - flask-restful==0.3.10 + - flask-socketio==5.3.6 + - flask-talisman==1.1.0 + - flatten-dict==0.4.2 + - fonttools==4.51.0 + - fqdn==1.5.1 + - frozenlist==1.4.1 + - fsspec==2024.3.1 + - future==1.0.0 + - g2p==2.0.0 + - gitdb==4.0.11 + - gitpython==3.1.43 + - greenlet==3.0.3 + - grpcio==1.63.0 + - h11==0.14.0 + - httpcore==1.0.5 + - httpx==0.27.0 + - huggingface-hub==0.23.0 + - humanfriendly==10.0 + - idna==3.7 + - importlib-resources==6.4.0 + - ipdb==0.13.13 + - ipykernel==6.29.4 + - ipython==8.24.0 + - isoduration==20.11.0 + - itsdangerous==2.2.0 + - jedi==0.19.1 + - jinja2==3.1.4 + - jiwer==3.0.4 + - joblib==1.4.2 + - json5==0.9.25 + - jsonpointer==2.4 + - jsonschema==4.22.0 + - jsonschema-specifications==2023.12.1 + - julius==0.2.7 + - jupyter-client==8.6.1 + - jupyter-core==5.7.2 + - jupyter-events==0.10.0 + - jupyter-lsp==2.2.5 + - jupyter-server==2.14.0 + - jupyter-server-terminals==0.5.3 + - jupyterlab==4.2.0 + - jupyterlab-pygments==0.3.0 + - jupyterlab-server==2.27.1 + - kiwisolver==1.4.5 + - lazy-loader==0.4 + - librosa==0.10.2 + - llvmlite==0.42.0 + - markdown==3.6 + - markdown-it-py==3.0.0 + - markdown2==2.4.13 + - markupsafe==2.1.5 + - matplotlib==3.8.4 + - matplotlib-inline==0.1.7 + - mdurl==0.1.2 + - mistune==3.0.2 + - mpmath==1.3.0 + - msgpack==1.0.8 + - multidict==6.0.5 + - multiprocess==0.70.16 + - munkres==1.1.4 + - nbclient==0.10.0 + - nbconvert==7.16.4 + - nbformat==5.10.4 + - nest-asyncio==1.6.0 + - networkx==3.3 + - notebook-shim==0.2.4 + - numba==0.59.1 + - numpy==1.26.4 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu12==2.20.5 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu12==12.1.105 + - openpyxl==3.1.2 + - overrides==7.7.0 + - packaging==24.0 + - pandas==2.2.2 + - pandocfilters==1.5.1 + - panphon==0.20.0 + - parler-tts==0.1 + - parso==0.8.4 + - pexpect==4.9.0 + - pillow==10.3.0 + - platformdirs==4.2.1 + - pooch==1.8.1 + - prometheus-client==0.20.0 + - prompt-toolkit==3.0.43 + - protobuf==3.19.6 + - psutil==5.9.8 + - ptyprocess==0.7.0 + - pure-eval==0.2.2 + - pyarrow==16.0.0 + - pyarrow-hotfix==0.6 + - pycparser==2.22 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pygments==2.18.0 + - pyloudnorm==0.1.1 + - pyparsing==3.1.2 + - pystoi==0.4.1 + - python-dateutil==2.9.0.post0 + - python-engineio==4.9.0 + - python-json-logger==2.0.7 + - python-socketio==5.11.2 + - pytz==2024.1 + - pyyaml==6.0.1 + - pyzmq==26.0.3 + - randomname==0.2.1 + - rapidfuzz==3.9.0 + - referencing==0.35.1 + - regex==2024.4.28 + - requests==2.31.0 + - rfc3339-validator==0.1.4 + - rfc3986-validator==0.1.1 + - rich==13.7.1 + - rpds-py==0.18.1 + - safetensors==0.4.3 + - scikit-learn==1.4.2 + - scipy==1.13.0 + - send2trash==1.8.3 + - sentencepiece==0.2.0 + - sentry-sdk==2.1.1 + - setproctitle==1.3.3 + - simple-websocket==1.0.0 + - six==1.16.0 + - smmap==5.0.1 + - sniffio==1.3.1 + - soundfile==0.12.1 + - soupsieve==2.5 + - soxr==0.3.7 + - stack-data==0.6.3 + - sympy==1.12 + - tensorboard==2.16.2 + - tensorboard-data-server==0.7.2 + - termcolor==2.4.0 + - terminado==0.18.1 + - text-unidecode==1.3 + - threadpoolctl==3.5.0 + - tinycss2==1.3.0 + - tokenizers==0.19.1 + - torch==2.3.0 + - torch-stoi==0.2.1 + - torchaudio==2.3.0 + - tornado==6.4 + - tqdm==4.66.4 + - traitlets==5.14.3 + - transformers==4.41.0.dev0 + - triton==2.3.0 + - types-python-dateutil==2.9.0.20240316 + - typing-extensions==4.11.0 + - tzdata==2024.1 + - unicodecsv==0.14.1 + - uri-template==1.3.0 + - urllib3==2.2.1 + - wandb==0.17.0 + - wcwidth==0.2.13 + - webcolors==1.13 + - webencodings==0.5.1 + - websocket-client==1.8.0 + - werkzeug==3.0.3 + - wsproto==1.2.0 + - xxhash==3.4.1 + - yarl==1.9.4 +prefix: /home/sanchit/miniconda3/envs/venv diff --git a/wandb/run-20240513_204652-m0g0ap7d/files/config.yaml b/wandb/run-20240513_204652-m0g0ap7d/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0343a1f8236adfe823f7154b75d9de813fe9b9a --- /dev/null +++ b/wandb/run-20240513_204652-m0g0ap7d/files/config.yaml @@ -0,0 +1,86 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.11.9 + cli_version: 0.17.0 + framework: huggingface + huggingface_version: 4.41.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1715626012 + t: + 1: + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 100 + 2: + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 100 + 3: + - 23 + 4: 3.11.9 + 5: 0.17.0 + 6: 4.41.0.dev0 + 8: + - 5 + 13: linux-x86_64 +learning_rate: + desc: null + value: 8.0e-05 +model_name_or_path: + desc: null + value: parler-tts/parler_tts_mini_v0.1 +num_train_epochs: + desc: null + value: 8 +gradient_accumulation_steps: + desc: null + value: 8 +per_device_train_batch_size: + desc: null + value: 16 +global_batch_size: + desc: null + value: 16 +mixed_precision: + desc: null + value: bf16 +lr_scheduler_type: + desc: null + value: SchedulerType.COSINE +warmup_steps: + desc: null + value: 250 +freeze_text_encoder: + desc: null + value: true +max_duration_in_seconds: + desc: null + value: 30.0 +weight_decay: + desc: null + value: 0.01 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.99 +temperature: + desc: null + value: 1.0 diff --git a/wandb/run-20240513_204652-m0g0ap7d/files/output.log b/wandb/run-20240513_204652-m0g0ap7d/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..dd550124d0275a70a71d5cf5a434f0bc7c355f7f --- /dev/null +++ b/wandb/run-20240513_204652-m0g0ap7d/files/output.log @@ -0,0 +1,180 @@ +05/13/2024 20:46:55 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, 16-bits training: False +05/13/2024 20:46:55 - INFO - __main__ - Training/evaluation parameters ParlerTTSTrainingArguments( +_n_gpu=1, +accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, +adafactor=False, +adam_beta1=0.9, +adam_beta2=0.99, +adam_epsilon=1e-08, +audio_encoder_per_device_batch_size=4, +auto_find_batch_size=False, +batch_eval_metrics=False, +bf16=False, +bf16_full_eval=False, +data_seed=None, +dataloader_drop_last=False, +dataloader_num_workers=4, +dataloader_persistent_workers=False, +dataloader_pin_memory=True, +dataloader_prefetch_factor=None, +ddp_backend=None, +ddp_broadcast_buffers=None, +ddp_bucket_cap_mb=None, +ddp_find_unused_parameters=None, +ddp_timeout=1800, +debug=[], +deepspeed=None, +disable_tqdm=False, +dispatch_batches=None, +do_eval=True, +do_predict=False, +do_train=True, +dtype=bfloat16, +eval_accumulation_steps=None, +eval_delay=0, +eval_do_concat_batches=True, +eval_steps=None, +eval_strategy=IntervalStrategy.EPOCH, +evaluation_strategy=epoch, +fp16=False, +fp16_backend=auto, +fp16_full_eval=False, +fp16_opt_level=O1, +fsdp=[], +fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, +fsdp_min_num_params=0, +fsdp_transformer_layer_cls_to_wrap=None, +full_determinism=False, +generation_config=None, +generation_max_length=None, +generation_num_beams=None, +gradient_accumulation_steps=8, +gradient_checkpointing=True, +gradient_checkpointing_kwargs=None, +greater_is_better=None, +group_by_length=True, +half_precision_backend=auto, +hub_always_push=False, +hub_model_id=None, +hub_private_repo=False, +hub_strategy=HubStrategy.EVERY_SAVE, +hub_token=, +ignore_data_skip=False, +include_inputs_for_metrics=True, +include_num_input_tokens_seen=False, +include_tokens_per_second=False, +jit_mode_eval=False, +label_names=None, +label_smoothing_factor=0.0, +learning_rate=8e-05, +length_column_name=length, +load_best_model_at_end=False, +local_rank=0, +log_level=passive, +log_level_replica=warning, +log_on_each_node=True, +logging_dir=../output_dir_training_constant_concat/runs/May13_20-46-51_hf-dgx-01, +logging_first_step=False, +logging_nan_inf_filter=True, +logging_steps=5, +logging_strategy=IntervalStrategy.STEPS, +lr_scheduler_kwargs={}, +lr_scheduler_type=SchedulerType.COSINE, +max_grad_norm=1.0, +max_steps=-1, +metric_for_best_model=None, +mp_parameters=, +neftune_noise_alpha=None, +no_cuda=False, +num_train_epochs=8, +optim=OptimizerNames.ADAMW_TORCH, +optim_args=None, +optim_target_modules=None, +output_dir=../output_dir_training_constant_concat/, +overwrite_output_dir=True, +past_index=-1, +per_device_eval_batch_size=16, +per_device_train_batch_size=16, +predict_with_generate=True, +prediction_loss_only=False, +push_to_hub=False, +push_to_hub_model_id=None, +push_to_hub_organization=None, +push_to_hub_token=, +ray_scope=last, +remove_unused_columns=True, +report_to=['wandb'], +restore_callback_states_from_checkpoint=False, +resume_from_checkpoint=None, +run_name=../output_dir_training_constant_concat/, +save_on_each_node=False, +save_only_model=False, +save_safetensors=True, +save_steps=500, +save_strategy=IntervalStrategy.EPOCH, +save_total_limit=5, +seed=456, +skip_memory_metrics=True, +sortish_sampler=False, +split_batches=None, +tf32=None, +torch_compile=False, +torch_compile_backend=None, +torch_compile_mode=None, +torchdynamo=None, +tpu_metrics_debug=False, +tpu_num_cores=None, +use_cpu=False, +use_ipex=False, +use_legacy_prediction_loop=False, +use_mps_device=False, +warmup_ratio=0.0, +warmup_steps=250, +weight_decay=0.01, +) +05/13/2024 20:46:57 - WARNING - __main__ - Disabling fast tokenizer warning: https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py#L3231-L3235 +loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--parler-tts--dac_44khZ_8kbps/snapshots/db52bea859d9411e0beb44a3ea923a8731ee4197/preprocessor_config.json +Feature extractor EncodecFeatureExtractor { + "chunk_length_s": null, + "feature_extractor_type": "EncodecFeatureExtractor", + "feature_size": 1, + "overlap": null, + "padding_side": "right", + "padding_value": 0.0, + "return_attention_mask": true, + "sampling_rate": 44100 +} +loading file spiece.model from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/spiece.model +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/tokenizer.json +loading file added_tokens.json from cache at None +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/tokenizer_config.json +You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers +loading file spiece.model from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/spiece.model +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/tokenizer.json +loading file added_tokens.json from cache at None +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/tokenizer_config.json +Combining datasets...: 0%| | 0/4 [00:00 + main() + File "/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py", line 950, in main + raw_datasets["train"] = load_multiple_datasets( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py", line 693, in load_multiple_datasets + metadata_dataset = load_dataset( + ^^^^^^^^^^^^^ + File "/home/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/datasets/load.py", line 2587, in load_dataset + builder_instance = load_dataset_builder( + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/datasets/load.py", line 2296, in load_dataset_builder + builder_instance: DatasetBuilder = builder_cls( + ^^^^^^^^^^^^ + File "/home/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/datasets/builder.py", line 374, in __init__ + self.config, self.config_id = self._create_builder_config( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/datasets/builder.py", line 599, in _create_builder_config + raise ValueError( +ValueError: BuilderConfig 'read' not found. Available: ['default'] \ No newline at end of file diff --git a/wandb/run-20240513_204652-m0g0ap7d/files/requirements.txt b/wandb/run-20240513_204652-m0g0ap7d/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..dc3da8d76890b37da99d889b5976fd39cb7ff10a --- /dev/null +++ b/wandb/run-20240513_204652-m0g0ap7d/files/requirements.txt @@ -0,0 +1,225 @@ +Babel==2.15.0 +Flask-Cors==4.0.1 +Flask-RESTful==0.3.10 +Flask-SocketIO==5.3.6 +Flask==2.2.5 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +PyYAML==6.0.1 +Pygments==2.18.0 +Send2Trash==1.8.3 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.30.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +aniso8601==9.0.1 +annotated-types==0.6.0 +anyio==4.3.0 +argbind==0.3.7 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +arrow==1.3.0 +asttokens==2.4.1 +async-lru==2.0.4 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bidict==0.23.1 +bitsandbytes==0.43.1 +bleach==6.1.0 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +coloredlogs==14.0 +comm==0.2.2 +contourpy==1.2.1 +cycler==0.12.1 +datasets==2.19.1 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +descript-audio-codec==1.0.0 +descript-audiotools==0.7.2 +dill==0.3.8 +dnspython==2.3.0 +docker-pycreds==0.4.0 +docstring_parser==0.16 +editdistance==0.8.1 +einops==0.8.0 +et-xmlfile==1.1.0 +evaluate==0.4.2 +eventlet==0.36.1 +executing==2.0.1 +fastjsonschema==2.19.1 +ffmpy==0.3.2 +filelock==3.14.0 +fire==0.6.0 +flask-talisman==1.1.0 +flatten-dict==0.4.2 +fonttools==4.51.0 +fqdn==1.5.1 +frozenlist==1.4.1 +fsspec==2024.3.1 +future==1.0.0 +g2p==2.0.0 +gitdb==4.0.11 +greenlet==3.0.3 +grpcio==1.63.0 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.0 +humanfriendly==10.0 +idna==3.7 +importlib_resources==6.4.0 +ipdb==0.13.13 +ipykernel==6.29.4 +ipython==8.24.0 +isoduration==20.11.0 +itsdangerous==2.2.0 +jedi==0.19.1 +jiwer==3.0.4 +joblib==1.4.2 +json5==0.9.25 +jsonpointer==2.4 +jsonschema-specifications==2023.12.1 +jsonschema==4.22.0 +julius==0.2.7 +jupyter-events==0.10.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.1 +jupyter_core==5.7.2 +jupyter_server==2.14.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.2.0 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.1 +kiwisolver==1.4.5 +lazy_loader==0.4 +librosa==0.10.2 +llvmlite==0.42.0 +markdown-it-py==3.0.0 +markdown2==2.4.13 +matplotlib-inline==0.1.7 +matplotlib==3.8.4 +mdurl==0.1.2 +mistune==3.0.2 +mpmath==1.3.0 +msgpack==1.0.8 +multidict==6.0.5 +multiprocess==0.70.16 +munkres==1.1.4 +nbclient==0.10.0 +nbconvert==7.16.4 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.3 +notebook_shim==0.2.4 +numba==0.59.1 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.1.105 +openpyxl==3.1.2 +overrides==7.7.0 +packaging==24.0 +pandas==2.2.2 +pandocfilters==1.5.1 +panphon==0.20.0 +parler_tts==0.1 +parso==0.8.4 +pexpect==4.9.0 +pillow==10.3.0 +pip==24.0 +platformdirs==4.2.1 +pooch==1.8.1 +prometheus_client==0.20.0 +prompt-toolkit==3.0.43 +protobuf==3.19.6 +psutil==5.9.8 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==16.0.0 +pycparser==2.22 +pydantic==2.7.1 +pydantic_core==2.18.2 +pyloudnorm==0.1.1 +pyparsing==3.1.2 +pystoi==0.4.1 +python-dateutil==2.9.0.post0 +python-engineio==4.9.0 +python-json-logger==2.0.7 +python-socketio==5.11.2 +pytz==2024.1 +pyzmq==26.0.3 +randomname==0.2.1 +rapidfuzz==3.9.0 +referencing==0.35.1 +regex==2024.4.28 +requests==2.31.0 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==13.7.1 +rpds-py==0.18.1 +safetensors==0.4.3 +scikit-learn==1.4.2 +scipy==1.13.0 +sentencepiece==0.2.0 +sentry-sdk==2.1.1 +setproctitle==1.3.3 +setuptools==69.5.1 +simple-websocket==1.0.0 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.1 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +stack-data==0.6.3 +sympy==1.12 +tensorboard-data-server==0.7.2 +tensorboard==2.16.2 +termcolor==2.4.0 +terminado==0.18.1 +text-unidecode==1.3 +threadpoolctl==3.5.0 +tinycss2==1.3.0 +tokenizers==0.19.1 +torch-stoi==0.2.1 +torch==2.3.0 +torchaudio==2.3.0 +tornado==6.4 +tqdm==4.66.4 +traitlets==5.14.3 +transformers==4.41.0.dev0 +transformers==4.41.0.dev0 +triton==2.3.0 +types-python-dateutil==2.9.0.20240316 +typing_extensions==4.11.0 +tzdata==2024.1 +unicodecsv==0.14.1 +uri-template==1.3.0 +urllib3==2.2.1 +wandb==0.17.0 +wcwidth==0.2.13 +webcolors==1.13 +webencodings==0.5.1 +websocket-client==1.8.0 +wheel==0.43.0 +wsproto==1.2.0 +xxhash==3.4.1 +yarl==1.9.4 \ No newline at end of file diff --git a/wandb/run-20240513_204652-m0g0ap7d/files/wandb-metadata.json b/wandb/run-20240513_204652-m0g0ap7d/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b18785e8d92680e759ba99fefae81eddda6e8d18 --- /dev/null +++ b/wandb/run-20240513_204652-m0g0ap7d/files/wandb-metadata.json @@ -0,0 +1,706 @@ +{ + "os": "Linux-5.4.0-166-generic-x86_64-with-glibc2.31", + "python": "3.11.9", + "heartbeatAt": "2024-05-13T18:46:53.365083", + "startedAt": "2024-05-13T18:46:52.816759", + "docker": null, + "cuda": null, + "args": [ + "finetuning_concatenated_config.json" + ], + "state": "running", + "program": "/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py", + "codePathLocal": "run_parler_tts_training.py", + "codePath": "run_parler_tts_training.py", + "git": { + "remote": "https://huggingface.co/sanchit-gandhi/parler-tts-mini-v0.1-expresso-concatenated-combined", + "commit": "50ba4323d7b8bb052629aa1b88283b9df081a821" + }, + "email": "sanchit@huggingface.co", + "root": "/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined", + "host": "hf-dgx-01", + "username": "sanchit", + "executable": "/home/sanchit/miniconda3/envs/venv/bin/python", + "cpu_count": 64, + "cpu_count_logical": 128, + "cpu_freq": { + "current": 2257.736234375, + "min": 1500.0, + "max": 2250.0 + }, + "cpu_freq_per_core": [ + { + "current": 1795.281, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1794.292, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1795.78, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1792.55, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1742.094, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3026.54, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1786.214, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1742.547, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1728.916, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1734.023, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3195.219, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1733.722, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3341.911, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3325.608, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3233.258, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1669.847, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1791.948, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1796.794, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1791.49, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1793.945, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3342.943, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1669.791, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1669.593, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1694.312, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1873.727, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1724.813, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2354.471, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1718.662, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1670.588, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1665.577, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1616.671, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2080.81, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1670.666, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1652.559, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3323.654, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1671.311, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1726.286, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1670.365, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3320.57, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1669.941, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1791.021, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1796.246, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1793.946, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1794.848, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3339.327, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3344.315, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3338.901, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1668.541, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1794.526, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1792.886, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1796.844, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1793.81, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1724.861, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2294.458, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1720.835, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1720.155, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1668.96, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1976.5, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2241.578, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1671.964, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3319.623, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1670.777, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1670.389, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1669.629, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1794.19, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1794.138, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1796.317, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1792.821, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1794.716, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1793.624, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1796.346, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1793.897, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1735.424, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1738.64, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1979.998, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1737.286, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3313.748, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3337.223, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1671.416, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1670.005, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1794.276, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1738.22, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1742.737, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1770.535, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3320.252, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1671.037, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1669.549, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1670.948, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2843.391, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2348.589, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3287.915, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2340.192, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2426.358, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2415.833, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2419.416, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2277.433, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2365.562, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2400.6, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2075.143, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2382.295, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3066.339, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2466.631, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3100.81, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2421.93, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3233.829, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2234.583, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2452.089, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2975.985, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3301.512, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3336.905, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2984.87, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2384.306, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2965.197, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1929.067, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1986.731, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1999.412, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2477.541, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3111.851, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2009.907, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1993.784, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2144.459, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3337.426, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3320.114, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2169.719, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3308.644, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2111.633, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2123.71, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2153.49, + "min": 1500.0, + "max": 2250.0 + } + ], + "disk": { + "/": { + "total": 1757.8785285949707, + "used": 1663.5005989074707 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 5, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA DGX Display", + "memory_total": 4294967296 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 503.5396919250488 + } +} diff --git a/wandb/run-20240513_204652-m0g0ap7d/files/wandb-summary.json b/wandb/run-20240513_204652-m0g0ap7d/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ccb0a228463b7c4b3058874db7eac137a7eef305 --- /dev/null +++ b/wandb/run-20240513_204652-m0g0ap7d/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 219}} \ No newline at end of file diff --git a/wandb/run-20240513_204652-m0g0ap7d/logs/debug-internal.log b/wandb/run-20240513_204652-m0g0ap7d/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..60751436f92950ea5c6fbd676e236ff7e19d7f34 --- /dev/null +++ b/wandb/run-20240513_204652-m0g0ap7d/logs/debug-internal.log @@ -0,0 +1,455 @@ +2024-05-13 20:46:52,823 INFO StreamThr :1244775 [internal.py:wandb_internal():85] W&B internal server running at pid: 1244775, started at: 2024-05-13 20:46:52.823043 +2024-05-13 20:46:52,825 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status +2024-05-13 20:46:52,826 INFO WriterThread:1244775 [datastore.py:open_for_write():87] open: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/run-m0g0ap7d.wandb +2024-05-13 20:46:52,828 DEBUG SenderThread:1244775 [sender.py:send():378] send: header +2024-05-13 20:46:52,829 DEBUG SenderThread:1244775 [sender.py:send():378] send: run +2024-05-13 20:46:53,223 INFO SenderThread:1244775 [dir_watcher.py:__init__():211] watching files in: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files +2024-05-13 20:46:53,223 INFO SenderThread:1244775 [sender.py:_start_run_threads():1123] run started: m0g0ap7d with start time 1715626012.822173 +2024-05-13 20:46:53,229 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: check_version +2024-05-13 20:46:53,229 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: check_version +2024-05-13 20:46:53,293 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: run_start +2024-05-13 20:46:53,323 DEBUG HandlerThread:1244775 [system_info.py:__init__():26] System info init +2024-05-13 20:46:53,323 DEBUG HandlerThread:1244775 [system_info.py:__init__():41] System info init done +2024-05-13 20:46:53,323 INFO HandlerThread:1244775 [system_monitor.py:start():194] Starting system monitor +2024-05-13 20:46:53,323 INFO SystemMonitor:1244775 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-13 20:46:53,324 INFO HandlerThread:1244775 [system_monitor.py:probe():214] Collecting system info +2024-05-13 20:46:53,324 INFO SystemMonitor:1244775 [interfaces.py:start():188] Started cpu monitoring +2024-05-13 20:46:53,325 INFO SystemMonitor:1244775 [interfaces.py:start():188] Started disk monitoring +2024-05-13 20:46:53,325 INFO SystemMonitor:1244775 [interfaces.py:start():188] Started gpu monitoring +2024-05-13 20:46:53,327 INFO SystemMonitor:1244775 [interfaces.py:start():188] Started memory monitoring +2024-05-13 20:46:53,329 INFO SystemMonitor:1244775 [interfaces.py:start():188] Started network monitoring +2024-05-13 20:46:53,365 DEBUG HandlerThread:1244775 [system_info.py:probe():150] Probing system +2024-05-13 20:46:53,366 DEBUG HandlerThread:1244775 [system_info.py:_probe_git():135] Probing git +2024-05-13 20:46:53,372 DEBUG HandlerThread:1244775 [system_info.py:_probe_git():143] Probing git done +2024-05-13 20:46:53,372 DEBUG HandlerThread:1244775 [system_info.py:probe():198] Probing system done +2024-05-13 20:46:53,372 DEBUG HandlerThread:1244775 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.31', 'python': '3.11.9', 'heartbeatAt': '2024-05-13T18:46:53.365083', 'startedAt': '2024-05-13T18:46:52.816759', 'docker': None, 'cuda': None, 'args': ('finetuning_concatenated_config.json',), 'state': 'running', 'program': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py', 'codePathLocal': 'run_parler_tts_training.py', 'codePath': 'run_parler_tts_training.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/parler-tts-mini-v0.1-expresso-concatenated-combined', 'commit': '50ba4323d7b8bb052629aa1b88283b9df081a821'}, 'email': 'sanchit@huggingface.co', 'root': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/miniconda3/envs/venv/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2257.736234375, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 1795.281, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.292, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.78, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.55, 'min': 1500.0, 'max': 2250.0}, {'current': 1742.094, 'min': 1500.0, 'max': 2250.0}, {'current': 3026.54, 'min': 1500.0, 'max': 2250.0}, {'current': 1786.214, 'min': 1500.0, 'max': 2250.0}, {'current': 1742.547, 'min': 1500.0, 'max': 2250.0}, {'current': 1728.916, 'min': 1500.0, 'max': 2250.0}, {'current': 1734.023, 'min': 1500.0, 'max': 2250.0}, {'current': 3195.219, 'min': 1500.0, 'max': 2250.0}, {'current': 1733.722, 'min': 1500.0, 'max': 2250.0}, {'current': 3341.911, 'min': 1500.0, 'max': 2250.0}, {'current': 3325.608, 'min': 1500.0, 'max': 2250.0}, {'current': 3233.258, 'min': 1500.0, 'max': 2250.0}, {'current': 1669.847, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.948, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.794, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.49, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.945, 'min': 1500.0, 'max': 2250.0}, {'current': 3342.943, 'min': 1500.0, 'max': 2250.0}, {'current': 1669.791, 'min': 1500.0, 'max': 2250.0}, {'current': 1669.593, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.312, 'min': 1500.0, 'max': 2250.0}, {'current': 1873.727, 'min': 1500.0, 'max': 2250.0}, {'current': 1724.813, 'min': 1500.0, 'max': 2250.0}, {'current': 2354.471, 'min': 1500.0, 'max': 2250.0}, {'current': 1718.662, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.588, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.577, 'min': 1500.0, 'max': 2250.0}, {'current': 1616.671, 'min': 1500.0, 'max': 2250.0}, {'current': 2080.81, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.666, 'min': 1500.0, 'max': 2250.0}, {'current': 1652.559, 'min': 1500.0, 'max': 2250.0}, {'current': 3323.654, 'min': 1500.0, 'max': 2250.0}, {'current': 1671.311, 'min': 1500.0, 'max': 2250.0}, {'current': 1726.286, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.365, 'min': 1500.0, 'max': 2250.0}, {'current': 3320.57, 'min': 1500.0, 'max': 2250.0}, {'current': 1669.941, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.021, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.246, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.946, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.848, 'min': 1500.0, 'max': 2250.0}, {'current': 3339.327, 'min': 1500.0, 'max': 2250.0}, {'current': 3344.315, 'min': 1500.0, 'max': 2250.0}, {'current': 3338.901, 'min': 1500.0, 'max': 2250.0}, {'current': 1668.541, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.526, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.886, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.844, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.81, 'min': 1500.0, 'max': 2250.0}, {'current': 1724.861, 'min': 1500.0, 'max': 2250.0}, {'current': 2294.458, 'min': 1500.0, 'max': 2250.0}, {'current': 1720.835, 'min': 1500.0, 'max': 2250.0}, {'current': 1720.155, 'min': 1500.0, 'max': 2250.0}, {'current': 1668.96, 'min': 1500.0, 'max': 2250.0}, {'current': 1976.5, 'min': 1500.0, 'max': 2250.0}, {'current': 2241.578, 'min': 1500.0, 'max': 2250.0}, {'current': 1671.964, 'min': 1500.0, 'max': 2250.0}, {'current': 3319.623, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.777, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.389, 'min': 1500.0, 'max': 2250.0}, {'current': 1669.629, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.19, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.138, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.317, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.821, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.716, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.624, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.346, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.897, 'min': 1500.0, 'max': 2250.0}, {'current': 1735.424, 'min': 1500.0, 'max': 2250.0}, {'current': 1738.64, 'min': 1500.0, 'max': 2250.0}, {'current': 1979.998, 'min': 1500.0, 'max': 2250.0}, {'current': 1737.286, 'min': 1500.0, 'max': 2250.0}, {'current': 3313.748, 'min': 1500.0, 'max': 2250.0}, {'current': 3337.223, 'min': 1500.0, 'max': 2250.0}, {'current': 1671.416, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.005, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.276, 'min': 1500.0, 'max': 2250.0}, {'current': 1738.22, 'min': 1500.0, 'max': 2250.0}, {'current': 1742.737, 'min': 1500.0, 'max': 2250.0}, {'current': 1770.535, 'min': 1500.0, 'max': 2250.0}, {'current': 3320.252, 'min': 1500.0, 'max': 2250.0}, {'current': 1671.037, 'min': 1500.0, 'max': 2250.0}, {'current': 1669.549, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.948, 'min': 1500.0, 'max': 2250.0}, {'current': 2843.391, 'min': 1500.0, 'max': 2250.0}, {'current': 2348.589, 'min': 1500.0, 'max': 2250.0}, {'current': 3287.915, 'min': 1500.0, 'max': 2250.0}, {'current': 2340.192, 'min': 1500.0, 'max': 2250.0}, {'current': 2426.358, 'min': 1500.0, 'max': 2250.0}, {'current': 2415.833, 'min': 1500.0, 'max': 2250.0}, {'current': 2419.416, 'min': 1500.0, 'max': 2250.0}, {'current': 2277.433, 'min': 1500.0, 'max': 2250.0}, {'current': 2365.562, 'min': 1500.0, 'max': 2250.0}, {'current': 2400.6, 'min': 1500.0, 'max': 2250.0}, {'current': 2075.143, 'min': 1500.0, 'max': 2250.0}, {'current': 2382.295, 'min': 1500.0, 'max': 2250.0}, {'current': 3066.339, 'min': 1500.0, 'max': 2250.0}, {'current': 2466.631, 'min': 1500.0, 'max': 2250.0}, {'current': 3100.81, 'min': 1500.0, 'max': 2250.0}, {'current': 2421.93, 'min': 1500.0, 'max': 2250.0}, {'current': 3233.829, 'min': 1500.0, 'max': 2250.0}, {'current': 2234.583, 'min': 1500.0, 'max': 2250.0}, {'current': 2452.089, 'min': 1500.0, 'max': 2250.0}, {'current': 2975.985, 'min': 1500.0, 'max': 2250.0}, {'current': 3301.512, 'min': 1500.0, 'max': 2250.0}, {'current': 3336.905, 'min': 1500.0, 'max': 2250.0}, {'current': 2984.87, 'min': 1500.0, 'max': 2250.0}, {'current': 2384.306, 'min': 1500.0, 'max': 2250.0}, {'current': 2965.197, 'min': 1500.0, 'max': 2250.0}, {'current': 1929.067, 'min': 1500.0, 'max': 2250.0}, {'current': 1986.731, 'min': 1500.0, 'max': 2250.0}, {'current': 1999.412, 'min': 1500.0, 'max': 2250.0}, {'current': 2477.541, 'min': 1500.0, 'max': 2250.0}, {'current': 3111.851, 'min': 1500.0, 'max': 2250.0}, {'current': 2009.907, 'min': 1500.0, 'max': 2250.0}, {'current': 1993.784, 'min': 1500.0, 'max': 2250.0}, {'current': 2144.459, 'min': 1500.0, 'max': 2250.0}, {'current': 3337.426, 'min': 1500.0, 'max': 2250.0}, {'current': 3320.114, 'min': 1500.0, 'max': 2250.0}, {'current': 2169.719, 'min': 1500.0, 'max': 2250.0}, {'current': 3308.644, 'min': 1500.0, 'max': 2250.0}, {'current': 2111.633, 'min': 1500.0, 'max': 2250.0}, {'current': 2123.71, 'min': 1500.0, 'max': 2250.0}, {'current': 2153.49, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1663.5005989074707}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} +2024-05-13 20:46:53,372 INFO HandlerThread:1244775 [system_monitor.py:probe():224] Finished collecting system info +2024-05-13 20:46:53,372 INFO HandlerThread:1244775 [system_monitor.py:probe():227] Publishing system info +2024-05-13 20:46:53,372 DEBUG HandlerThread:1244775 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-13 20:46:53,387 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:46:53,400 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:46:54,224 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/conda-environment.yaml +2024-05-13 20:46:55,418 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:46:55,429 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:46:55,741 DEBUG HandlerThread:1244775 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-13 20:46:55,742 INFO HandlerThread:1244775 [system_monitor.py:probe():229] Finished publishing system info +2024-05-13 20:46:55,750 DEBUG SenderThread:1244775 [sender.py:send():378] send: files +2024-05-13 20:46:55,750 INFO SenderThread:1244775 [sender.py:_save_file():1389] saving file wandb-metadata.json with policy now +2024-05-13 20:46:55,863 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: python_packages +2024-05-13 20:46:55,863 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:46:55,863 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: python_packages +2024-05-13 20:46:55,865 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:46:56,093 DEBUG SenderThread:1244775 [sender.py:send():378] send: telemetry +2024-05-13 20:46:56,093 DEBUG SenderThread:1244775 [sender.py:send():378] send: config +2024-05-13 20:46:56,224 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/conda-environment.yaml +2024-05-13 20:46:56,224 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/wandb-metadata.json +2024-05-13 20:46:56,224 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/requirements.txt +2024-05-13 20:46:56,224 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/output.log +2024-05-13 20:46:56,261 INFO wandb-upload_0:1244775 [upload_job.py:push():130] Uploaded file /tmp/tmpewnm1an9wandb/l0duo91p-wandb-metadata.json +2024-05-13 20:46:58,225 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/output.log +2024-05-13 20:46:58,331 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:46:58,343 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:46:58,386 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:00,362 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:00,375 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:03,387 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:03,431 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:03,443 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:05,466 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:05,478 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:08,388 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:08,679 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:08,692 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:10,713 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:10,724 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:10,863 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:47:10,863 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:47:12,746 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:12,757 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:14,094 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:15,627 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:15,637 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:17,658 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:17,668 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:19,096 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:20,779 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:20,799 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:22,817 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:22,830 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:24,099 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:24,858 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:24,870 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:25,233 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/config.yaml +2024-05-13 20:47:25,863 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:47:25,863 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:47:27,736 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:27,747 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:30,092 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:31,414 ERROR gpu :1244775 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:47:31,434 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:31,462 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:34,494 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:34,518 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:35,093 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:36,569 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:36,599 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:38,635 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:38,658 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:40,093 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:40,863 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:47:40,864 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:47:41,560 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:41,584 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:43,631 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:43,652 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:46,084 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:46,570 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:46,604 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:48,647 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:48,659 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:51,084 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:51,664 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:51,686 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:53,329 DEBUG SystemMonitor:1244775 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-13 20:47:53,333 DEBUG SenderThread:1244775 [sender.py:send():378] send: stats +2024-05-13 20:47:53,709 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:53,724 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:55,863 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:47:55,864 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:47:56,593 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:56,607 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:57,080 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:47:58,627 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:47:58,641 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:01,653 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:01,664 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:02,081 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:48:03,684 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:03,696 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:06,665 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:06,679 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:07,082 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:48:10,344 ERROR gpu :1244775 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:48:10,366 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:10,380 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:10,863 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:48:10,864 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:48:13,048 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:48:13,506 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:13,529 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:15,558 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:15,586 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:18,050 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:48:18,552 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:18,572 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:20,626 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:20,644 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:23,050 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:48:23,336 DEBUG SenderThread:1244775 [sender.py:send():378] send: stats +2024-05-13 20:48:23,683 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:23,707 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:25,750 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:25,769 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:25,863 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:48:25,864 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:48:28,681 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:28,701 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:29,005 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:48:30,725 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:30,747 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:33,782 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:33,801 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:34,006 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:48:35,835 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:35,858 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:38,877 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:38,904 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:39,007 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:48:40,863 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:48:40,864 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:48:40,932 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:40,946 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:42,969 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:42,980 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:44,036 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:48:45,801 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:45,831 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:49,037 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:48:49,512 ERROR gpu :1244775 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:48:49,550 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:49,567 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:52,479 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:52,494 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:53,338 DEBUG SenderThread:1244775 [sender.py:send():378] send: stats +2024-05-13 20:48:54,339 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:48:54,518 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:54,530 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:55,864 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:48:55,864 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:48:57,576 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:57,589 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:59,615 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:48:59,628 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:00,007 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:02,716 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:02,730 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:04,748 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:04,763 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:05,008 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:07,719 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:07,741 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:09,766 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:09,776 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:10,009 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:10,864 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:49:10,864 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:49:12,861 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:12,875 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:14,916 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:14,934 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:15,085 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:17,938 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:17,957 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:20,002 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:20,012 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:20,086 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:22,621 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:22,641 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:23,339 DEBUG SenderThread:1244775 [sender.py:send():378] send: stats +2024-05-13 20:49:24,675 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:24,686 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:25,341 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:25,864 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:49:25,865 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:49:28,370 ERROR gpu :1244775 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:49:28,612 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:28,633 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:30,656 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:30,678 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:31,038 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:33,121 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:33,150 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:35,177 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:35,217 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:36,039 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:37,932 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:37,967 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:40,002 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:40,032 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:40,864 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:49:40,864 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:49:42,037 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:42,295 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:42,620 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:44,641 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:44,688 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:47,037 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:47,242 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:47,286 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:49,336 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:49,362 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:51,898 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:51,927 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:52,038 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:53,343 DEBUG SenderThread:1244775 [sender.py:send():378] send: stats +2024-05-13 20:49:54,378 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:54,396 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:55,864 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:49:55,864 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:49:56,417 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:56,465 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:58,028 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:49:58,997 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:49:59,011 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:01,057 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:01,097 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:03,029 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:50:05,200 ERROR gpu :1244775 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:50:05,253 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:05,272 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:07,311 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:07,330 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:08,029 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:50:09,887 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:09,918 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:10,864 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:50:10,864 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:50:11,934 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:11,942 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:13,047 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:50:14,680 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:14,697 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:16,719 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:16,738 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:18,047 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:50:19,232 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:19,254 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:21,272 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:21,282 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:23,048 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:50:23,346 DEBUG SenderThread:1244775 [sender.py:send():378] send: stats +2024-05-13 20:50:23,802 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:23,817 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:25,839 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:25,850 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:25,864 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:50:25,865 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:50:28,061 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:50:28,256 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:28,265 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:30,284 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:30,296 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:32,293 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/output.log +2024-05-13 20:50:32,787 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:32,796 ERROR gpu :1244775 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:50:33,152 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:50:33,158 DEBUG SenderThread:1244775 [sender.py:send():378] send: exit +2024-05-13 20:50:33,158 INFO SenderThread:1244775 [sender.py:send_exit():585] handling exit code: 1 +2024-05-13 20:50:33,159 INFO SenderThread:1244775 [sender.py:send_exit():587] handling runtime: 219 +2024-05-13 20:50:33,159 INFO SenderThread:1244775 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 20:50:33,159 INFO SenderThread:1244775 [sender.py:send_exit():593] send defer +2024-05-13 20:50:33,159 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:33,159 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 0 +2024-05-13 20:50:33,159 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:33,160 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 0 +2024-05-13 20:50:33,160 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 1 +2024-05-13 20:50:33,160 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:33,160 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 1 +2024-05-13 20:50:33,160 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:33,160 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 1 +2024-05-13 20:50:33,160 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 2 +2024-05-13 20:50:33,160 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:33,160 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 2 +2024-05-13 20:50:33,160 INFO HandlerThread:1244775 [system_monitor.py:finish():203] Stopping system monitor +2024-05-13 20:50:33,161 DEBUG SystemMonitor:1244775 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-13 20:50:33,161 DEBUG SystemMonitor:1244775 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-13 20:50:33,161 INFO HandlerThread:1244775 [interfaces.py:finish():200] Joined cpu monitor +2024-05-13 20:50:33,164 INFO HandlerThread:1244775 [interfaces.py:finish():200] Joined disk monitor +2024-05-13 20:50:33,293 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/wandb-summary.json +2024-05-13 20:50:34,293 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/output.log +2024-05-13 20:50:34,797 ERROR gpu :1244775 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:50:34,797 INFO HandlerThread:1244775 [interfaces.py:finish():200] Joined gpu monitor +2024-05-13 20:50:34,797 INFO HandlerThread:1244775 [interfaces.py:finish():200] Joined memory monitor +2024-05-13 20:50:34,797 INFO HandlerThread:1244775 [interfaces.py:finish():200] Joined network monitor +2024-05-13 20:50:34,798 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 20:50:34,799 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:34,799 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 2 +2024-05-13 20:50:34,799 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 3 +2024-05-13 20:50:34,800 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:34,800 DEBUG SenderThread:1244775 [sender.py:send():378] send: stats +2024-05-13 20:50:34,800 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 3 +2024-05-13 20:50:34,800 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 20:50:34,801 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:34,801 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 3 +2024-05-13 20:50:34,801 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 4 +2024-05-13 20:50:34,801 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:34,801 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 4 +2024-05-13 20:50:34,801 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:34,801 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 4 +2024-05-13 20:50:34,801 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 5 +2024-05-13 20:50:34,802 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:34,802 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 5 +2024-05-13 20:50:34,802 DEBUG SenderThread:1244775 [sender.py:send():378] send: summary +2024-05-13 20:50:34,802 INFO SenderThread:1244775 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 20:50:34,802 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:34,802 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 5 +2024-05-13 20:50:34,802 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 6 +2024-05-13 20:50:34,802 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:34,802 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 6 +2024-05-13 20:50:34,803 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:34,803 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 6 +2024-05-13 20:50:34,803 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 7 +2024-05-13 20:50:34,803 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:50:34,803 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:34,803 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 7 +2024-05-13 20:50:34,803 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:34,803 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 7 +2024-05-13 20:50:35,159 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 20:50:35,294 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/wandb-summary.json +2024-05-13 20:50:38,152 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 8 +2024-05-13 20:50:38,152 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 20:50:38,152 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:38,153 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 8 +2024-05-13 20:50:38,153 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:38,153 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 8 +2024-05-13 20:50:38,153 INFO SenderThread:1244775 [job_builder.py:build():432] Attempting to build job artifact +2024-05-13 20:50:38,153 INFO SenderThread:1244775 [job_builder.py:_get_source_type():565] is repo sourced job +2024-05-13 20:50:38,160 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 20:50:38,179 INFO SenderThread:1244775 [job_builder.py:build():541] adding wandb-job metadata file +2024-05-13 20:50:38,181 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 9 +2024-05-13 20:50:38,181 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 20:50:38,181 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:38,182 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 9 +2024-05-13 20:50:38,182 DEBUG SenderThread:1244775 [sender.py:send():378] send: artifact +2024-05-13 20:50:38,294 INFO Thread-12 :1244775 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/output.log +2024-05-13 20:50:39,160 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 20:50:39,209 INFO wandb-upload_0:1244775 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmp34vs1_ku +2024-05-13 20:50:39,238 INFO wandb-upload_1:1244775 [upload_job.py:push():88] Uploaded file /tmp/tmp_ne7l6g3/wandb-job.json +2024-05-13 20:50:40,085 INFO SenderThread:1244775 [sender.py:send_artifact():1467] sent artifact job-https___huggingface.co_sanchit-gandhi_parler-tts-mini-v0.1-expresso-concatenated-combined_run_parler_tts_training.py - {'id': 'QXJ0aWZhY3Q6ODM0NzI5NzMx', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDIzMTI3NQ==', 'latestArtifact': None}} +2024-05-13 20:50:40,086 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:40,086 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 9 +2024-05-13 20:50:40,086 INFO SenderThread:1244775 [dir_watcher.py:finish():358] shutting down directory watcher +2024-05-13 20:50:40,295 INFO SenderThread:1244775 [dir_watcher.py:finish():388] scan: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files +2024-05-13 20:50:40,295 INFO SenderThread:1244775 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/conda-environment.yaml conda-environment.yaml +2024-05-13 20:50:40,295 INFO SenderThread:1244775 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/wandb-summary.json wandb-summary.json +2024-05-13 20:50:40,295 INFO SenderThread:1244775 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/output.log output.log +2024-05-13 20:50:40,297 INFO SenderThread:1244775 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/requirements.txt requirements.txt +2024-05-13 20:50:40,300 INFO SenderThread:1244775 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/config.yaml config.yaml +2024-05-13 20:50:40,301 INFO SenderThread:1244775 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/wandb-metadata.json wandb-metadata.json +2024-05-13 20:50:40,301 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 10 +2024-05-13 20:50:40,301 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 20:50:40,301 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:40,302 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 10 +2024-05-13 20:50:40,306 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:40,306 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 10 +2024-05-13 20:50:40,306 INFO SenderThread:1244775 [file_pusher.py:finish():169] shutting down file pusher +2024-05-13 20:50:40,632 INFO wandb-upload_0:1244775 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/conda-environment.yaml +2024-05-13 20:50:40,639 INFO wandb-upload_1:1244775 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/output.log +2024-05-13 20:50:40,758 INFO wandb-upload_3:1244775 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/requirements.txt +2024-05-13 20:50:40,773 INFO wandb-upload_2:1244775 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/wandb-summary.json +2024-05-13 20:50:40,918 INFO wandb-upload_4:1244775 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/files/config.yaml +2024-05-13 20:50:41,118 INFO Thread-11 (_thread_body):1244775 [sender.py:transition_state():613] send defer: 11 +2024-05-13 20:50:41,119 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:41,119 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 11 +2024-05-13 20:50:41,119 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:41,119 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 11 +2024-05-13 20:50:41,120 INFO SenderThread:1244775 [file_pusher.py:join():175] waiting for file pusher +2024-05-13 20:50:41,120 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 12 +2024-05-13 20:50:41,120 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:41,120 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 12 +2024-05-13 20:50:41,120 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:41,120 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 12 +2024-05-13 20:50:41,120 INFO SenderThread:1244775 [file_stream.py:finish():601] file stream finish called +2024-05-13 20:50:41,161 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 20:50:41,510 INFO SenderThread:1244775 [file_stream.py:finish():605] file stream finish is done +2024-05-13 20:50:41,510 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 13 +2024-05-13 20:50:41,510 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 20:50:41,510 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:41,510 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 13 +2024-05-13 20:50:41,510 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:41,510 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 13 +2024-05-13 20:50:41,510 INFO SenderThread:1244775 [sender.py:transition_state():613] send defer: 14 +2024-05-13 20:50:41,511 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: defer +2024-05-13 20:50:41,511 INFO HandlerThread:1244775 [handler.py:handle_request_defer():184] handle defer: 14 +2024-05-13 20:50:41,511 DEBUG SenderThread:1244775 [sender.py:send():378] send: final +2024-05-13 20:50:41,511 DEBUG SenderThread:1244775 [sender.py:send():378] send: footer +2024-05-13 20:50:41,511 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: defer +2024-05-13 20:50:41,511 INFO SenderThread:1244775 [sender.py:send_request_defer():609] handle sender defer: 14 +2024-05-13 20:50:41,512 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 20:50:41,512 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 20:50:41,512 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 20:50:41,512 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 20:50:41,513 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: server_info +2024-05-13 20:50:41,513 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: get_summary +2024-05-13 20:50:41,513 DEBUG SenderThread:1244775 [sender.py:send_request():405] send_request: server_info +2024-05-13 20:50:41,515 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: sampled_history +2024-05-13 20:50:41,515 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: internal_messages +2024-05-13 20:50:41,651 INFO MainThread:1244775 [wandb_run.py:_footer_history_summary_info():3994] rendering history +2024-05-13 20:50:41,651 INFO MainThread:1244775 [wandb_run.py:_footer_history_summary_info():4026] rendering summary +2024-05-13 20:50:41,651 INFO MainThread:1244775 [wandb_run.py:_footer_sync_info():3953] logging synced files +2024-05-13 20:50:41,651 DEBUG HandlerThread:1244775 [handler.py:handle_request():158] handle_request: shutdown +2024-05-13 20:50:41,651 INFO HandlerThread:1244775 [handler.py:finish():882] shutting down handler +2024-05-13 20:50:42,513 INFO WriterThread:1244775 [datastore.py:close():296] close: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/run-m0g0ap7d.wandb +2024-05-13 20:50:42,651 INFO SenderThread:1244775 [sender.py:finish():1545] shutting down sender +2024-05-13 20:50:42,651 INFO SenderThread:1244775 [file_pusher.py:finish():169] shutting down file pusher +2024-05-13 20:50:42,651 INFO SenderThread:1244775 [file_pusher.py:join():175] waiting for file pusher diff --git a/wandb/run-20240513_204652-m0g0ap7d/logs/debug.log b/wandb/run-20240513_204652-m0g0ap7d/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2e0c83befd038fae45b350443b7c335be9476fe1 --- /dev/null +++ b/wandb/run-20240513_204652-m0g0ap7d/logs/debug.log @@ -0,0 +1,29 @@ +2024-05-13 20:46:52,817 INFO MainThread:1244577 [wandb_setup.py:_flush():76] Current SDK version is 0.17.0 +2024-05-13 20:46:52,817 INFO MainThread:1244577 [wandb_setup.py:_flush():76] Configure stats pid to 1244577 +2024-05-13 20:46:52,817 INFO MainThread:1244577 [wandb_setup.py:_flush():76] Loading settings from /home/sanchit/.config/wandb/settings +2024-05-13 20:46:52,817 INFO MainThread:1244577 [wandb_setup.py:_flush():76] Loading settings from /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/settings +2024-05-13 20:46:52,817 INFO MainThread:1244577 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-13 20:46:52,817 INFO MainThread:1244577 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-13 20:46:52,817 INFO MainThread:1244577 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'run_parler_tts_training.py', 'program_abspath': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py', 'program': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py'} +2024-05-13 20:46:52,817 INFO MainThread:1244577 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-13 20:46:52,817 INFO MainThread:1244577 [wandb_init.py:_log_setup():520] Logging user logs to /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/logs/debug.log +2024-05-13 20:46:52,818 INFO MainThread:1244577 [wandb_init.py:_log_setup():521] Logging internal logs to /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_204652-m0g0ap7d/logs/debug-internal.log +2024-05-13 20:46:52,818 INFO MainThread:1244577 [wandb_init.py:init():560] calling init triggers +2024-05-13 20:46:52,818 INFO MainThread:1244577 [wandb_init.py:init():567] wandb.init called with sweep_config: {} +config: {} +2024-05-13 20:46:52,818 INFO MainThread:1244577 [wandb_init.py:init():610] starting backend +2024-05-13 20:46:52,818 INFO MainThread:1244577 [wandb_init.py:init():614] setting up manager +2024-05-13 20:46:52,821 INFO MainThread:1244577 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-13 20:46:52,822 INFO MainThread:1244577 [wandb_init.py:init():622] backend started and connected +2024-05-13 20:46:52,823 INFO MainThread:1244577 [wandb_init.py:init():711] updated telemetry +2024-05-13 20:46:52,826 INFO MainThread:1244577 [wandb_init.py:init():744] communicating run to backend with 90.0 second timeout +2024-05-13 20:46:53,228 INFO MainThread:1244577 [wandb_run.py:_on_init():2396] communicating current version +2024-05-13 20:46:53,287 INFO MainThread:1244577 [wandb_run.py:_on_init():2405] got version response +2024-05-13 20:46:53,287 INFO MainThread:1244577 [wandb_init.py:init():795] starting run threads in backend +2024-05-13 20:46:55,863 INFO MainThread:1244577 [wandb_run.py:_console_start():2374] atexit reg +2024-05-13 20:46:55,863 INFO MainThread:1244577 [wandb_run.py:_redirect():2229] redirect: wrap_raw +2024-05-13 20:46:55,863 INFO MainThread:1244577 [wandb_run.py:_redirect():2294] Wrapping output streams. +2024-05-13 20:46:55,863 INFO MainThread:1244577 [wandb_run.py:_redirect():2319] Redirects installed. +2024-05-13 20:46:55,864 INFO MainThread:1244577 [wandb_init.py:init():838] run started, returning control to user process +2024-05-13 20:46:55,864 INFO MainThread:1244577 [wandb_run.py:_config_callback():1376] config_cb None None {'learning_rate': 8e-05, 'model_name_or_path': 'parler-tts/parler_tts_mini_v0.1', 'num_train_epochs': 8, 'gradient_accumulation_steps': 8, 'per_device_train_batch_size': 16, 'global_batch_size': 16, 'mixed_precision': 'bf16', 'lr_scheduler_type': 'SchedulerType.COSINE', 'warmup_steps': 250, 'freeze_text_encoder': True, 'max_duration_in_seconds': 30.0, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.99, 'temperature': 1.0} +2024-05-13 20:50:42,653 WARNING MsgRouterThr:1244577 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240513_204652-m0g0ap7d/run-m0g0ap7d.wandb b/wandb/run-20240513_204652-m0g0ap7d/run-m0g0ap7d.wandb new file mode 100644 index 0000000000000000000000000000000000000000..97577dbfc6f9738b73062e33cdfc741176b4e480 Binary files /dev/null and b/wandb/run-20240513_204652-m0g0ap7d/run-m0g0ap7d.wandb differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/conda-environment.yaml b/wandb/run-20240513_205249-qaoje1x9/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ecd1ac96b302b32274bdbd12ac08d3e1d65a777 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/conda-environment.yaml @@ -0,0 +1,248 @@ +name: venv +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - bzip2=1.0.8=h5eee18b_6 + - ca-certificates=2024.3.11=h06a4308_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_1 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - libuuid=1.41.5=h5eee18b_0 + - ncurses=6.4=h6a678d5_0 + - openssl=3.0.13=h7f8727e_1 + - pip=24.0=py311h06a4308_0 + - python=3.11.9=h955ad1f_0 + - readline=8.2=h5eee18b_0 + - setuptools=69.5.1=py311h06a4308_0 + - sqlite=3.45.3=h5eee18b_0 + - tk=8.6.14=h39e8969_0 + - wheel=0.43.0=py311h06a4308_0 + - xz=5.4.6=h5eee18b_1 + - zlib=1.2.13=h5eee18b_1 + - pip: + - absl-py==2.1.0 + - accelerate==0.30.0 + - aiohttp==3.9.5 + - aiosignal==1.3.1 + - aniso8601==9.0.1 + - annotated-types==0.6.0 + - anyio==4.3.0 + - argbind==0.3.7 + - argon2-cffi==23.1.0 + - argon2-cffi-bindings==21.2.0 + - arrow==1.3.0 + - asttokens==2.4.1 + - async-lru==2.0.4 + - attrs==23.2.0 + - audioread==3.0.1 + - babel==2.15.0 + - beautifulsoup4==4.12.3 + - bidict==0.23.1 + - bitsandbytes==0.43.1 + - bleach==6.1.0 + - certifi==2024.2.2 + - cffi==1.16.0 + - charset-normalizer==3.3.2 + - click==8.1.7 + - coloredlogs==14.0 + - comm==0.2.2 + - contourpy==1.2.1 + - cycler==0.12.1 + - datasets==2.19.1 + - debugpy==1.8.1 + - decorator==5.1.1 + - defusedxml==0.7.1 + - descript-audio-codec==1.0.0 + - descript-audiotools==0.7.2 + - dill==0.3.8 + - dnspython==2.3.0 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - editdistance==0.8.1 + - einops==0.8.0 + - et-xmlfile==1.1.0 + - evaluate==0.4.2 + - eventlet==0.36.1 + - executing==2.0.1 + - fastjsonschema==2.19.1 + - ffmpy==0.3.2 + - filelock==3.14.0 + - fire==0.6.0 + - flask==2.2.5 + - flask-cors==4.0.1 + - flask-restful==0.3.10 + - flask-socketio==5.3.6 + - flask-talisman==1.1.0 + - flatten-dict==0.4.2 + - fonttools==4.51.0 + - fqdn==1.5.1 + - frozenlist==1.4.1 + - fsspec==2024.3.1 + - future==1.0.0 + - g2p==2.0.0 + - gitdb==4.0.11 + - gitpython==3.1.43 + - greenlet==3.0.3 + - grpcio==1.63.0 + - h11==0.14.0 + - httpcore==1.0.5 + - httpx==0.27.0 + - huggingface-hub==0.23.0 + - humanfriendly==10.0 + - idna==3.7 + - importlib-resources==6.4.0 + - ipdb==0.13.13 + - ipykernel==6.29.4 + - ipython==8.24.0 + - isoduration==20.11.0 + - itsdangerous==2.2.0 + - jedi==0.19.1 + - jinja2==3.1.4 + - jiwer==3.0.4 + - joblib==1.4.2 + - json5==0.9.25 + - jsonpointer==2.4 + - jsonschema==4.22.0 + - jsonschema-specifications==2023.12.1 + - julius==0.2.7 + - jupyter-client==8.6.1 + - jupyter-core==5.7.2 + - jupyter-events==0.10.0 + - jupyter-lsp==2.2.5 + - jupyter-server==2.14.0 + - jupyter-server-terminals==0.5.3 + - jupyterlab==4.2.0 + - jupyterlab-pygments==0.3.0 + - jupyterlab-server==2.27.1 + - kiwisolver==1.4.5 + - lazy-loader==0.4 + - librosa==0.10.2 + - llvmlite==0.42.0 + - markdown==3.6 + - markdown-it-py==3.0.0 + - markdown2==2.4.13 + - markupsafe==2.1.5 + - matplotlib==3.8.4 + - matplotlib-inline==0.1.7 + - mdurl==0.1.2 + - mistune==3.0.2 + - mpmath==1.3.0 + - msgpack==1.0.8 + - multidict==6.0.5 + - multiprocess==0.70.16 + - munkres==1.1.4 + - nbclient==0.10.0 + - nbconvert==7.16.4 + - nbformat==5.10.4 + - nest-asyncio==1.6.0 + - networkx==3.3 + - notebook-shim==0.2.4 + - numba==0.59.1 + - numpy==1.26.4 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu12==2.20.5 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu12==12.1.105 + - openpyxl==3.1.2 + - overrides==7.7.0 + - packaging==24.0 + - pandas==2.2.2 + - pandocfilters==1.5.1 + - panphon==0.20.0 + - parler-tts==0.1 + - parso==0.8.4 + - pexpect==4.9.0 + - pillow==10.3.0 + - platformdirs==4.2.1 + - pooch==1.8.1 + - prometheus-client==0.20.0 + - prompt-toolkit==3.0.43 + - protobuf==3.19.6 + - psutil==5.9.8 + - ptyprocess==0.7.0 + - pure-eval==0.2.2 + - pyarrow==16.0.0 + - pyarrow-hotfix==0.6 + - pycparser==2.22 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pygments==2.18.0 + - pyloudnorm==0.1.1 + - pyparsing==3.1.2 + - pystoi==0.4.1 + - python-dateutil==2.9.0.post0 + - python-engineio==4.9.0 + - python-json-logger==2.0.7 + - python-socketio==5.11.2 + - pytz==2024.1 + - pyyaml==6.0.1 + - pyzmq==26.0.3 + - randomname==0.2.1 + - rapidfuzz==3.9.0 + - referencing==0.35.1 + - regex==2024.4.28 + - requests==2.31.0 + - rfc3339-validator==0.1.4 + - rfc3986-validator==0.1.1 + - rich==13.7.1 + - rpds-py==0.18.1 + - safetensors==0.4.3 + - scikit-learn==1.4.2 + - scipy==1.13.0 + - send2trash==1.8.3 + - sentencepiece==0.2.0 + - sentry-sdk==2.1.1 + - setproctitle==1.3.3 + - simple-websocket==1.0.0 + - six==1.16.0 + - smmap==5.0.1 + - sniffio==1.3.1 + - soundfile==0.12.1 + - soupsieve==2.5 + - soxr==0.3.7 + - stack-data==0.6.3 + - sympy==1.12 + - tensorboard==2.16.2 + - tensorboard-data-server==0.7.2 + - termcolor==2.4.0 + - terminado==0.18.1 + - text-unidecode==1.3 + - threadpoolctl==3.5.0 + - tinycss2==1.3.0 + - tokenizers==0.19.1 + - torch==2.3.0 + - torch-stoi==0.2.1 + - torchaudio==2.3.0 + - tornado==6.4 + - tqdm==4.66.4 + - traitlets==5.14.3 + - transformers==4.41.0.dev0 + - triton==2.3.0 + - types-python-dateutil==2.9.0.20240316 + - typing-extensions==4.11.0 + - tzdata==2024.1 + - unicodecsv==0.14.1 + - uri-template==1.3.0 + - urllib3==2.2.1 + - wandb==0.17.0 + - wcwidth==0.2.13 + - webcolors==1.13 + - webencodings==0.5.1 + - websocket-client==1.8.0 + - werkzeug==3.0.3 + - wsproto==1.2.0 + - xxhash==3.4.1 + - yarl==1.9.4 +prefix: /home/sanchit/miniconda3/envs/venv diff --git a/wandb/run-20240513_205249-qaoje1x9/files/config.yaml b/wandb/run-20240513_205249-qaoje1x9/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec236c6c333e03c443a63d4745ad131850a7515a --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/config.yaml @@ -0,0 +1,88 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.11.9 + cli_version: 0.17.0 + framework: huggingface + huggingface_version: 4.41.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1715626369 + t: + 1: + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 100 + 2: + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 100 + 3: + - 2 + - 23 + - 61 + 4: 3.11.9 + 5: 0.17.0 + 6: 4.41.0.dev0 + 8: + - 5 + 13: linux-x86_64 +learning_rate: + desc: null + value: 8.0e-05 +model_name_or_path: + desc: null + value: parler-tts/parler_tts_mini_v0.1 +num_train_epochs: + desc: null + value: 8 +gradient_accumulation_steps: + desc: null + value: 8 +per_device_train_batch_size: + desc: null + value: 16 +global_batch_size: + desc: null + value: 16 +mixed_precision: + desc: null + value: bf16 +lr_scheduler_type: + desc: null + value: SchedulerType.COSINE +warmup_steps: + desc: null + value: 250 +freeze_text_encoder: + desc: null + value: true +max_duration_in_seconds: + desc: null + value: 30.0 +weight_decay: + desc: null + value: 0.01 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.99 +temperature: + desc: null + value: 1.0 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav new file mode 100644 index 0000000000000000000000000000000000000000..151777a5a84cc57b944388d6b0e3dd8a2ac6e112 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav new file mode 100644 index 0000000000000000000000000000000000000000..f72f35fc55b359929ac2578f75f0acefc44edf12 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav new file mode 100644 index 0000000000000000000000000000000000000000..eed6998fc601fb3c676a1794a2e5116abec0d031 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav new file mode 100644 index 0000000000000000000000000000000000000000..976df927e37f4713493c9cf62d88427e60732351 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav new file mode 100644 index 0000000000000000000000000000000000000000..b84a465eb03623efa0a95ee9ed2fb103d0765799 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav new file mode 100644 index 0000000000000000000000000000000000000000..5380bf4233e27564000d2b8c4e4dd4153a511e23 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav new file mode 100644 index 0000000000000000000000000000000000000000..d6cd04bdf828454179c721409b9ef7f1c9ec8ba1 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav new file mode 100644 index 0000000000000000000000000000000000000000..1afd150794a4aca0dd580a9f9d2465684e4d090e Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav new file mode 100644 index 0000000000000000000000000000000000000000..b753f1a2125694d6cf6227ee2e54de18011f9450 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav new file mode 100644 index 0000000000000000000000000000000000000000..c36574567d6397522b6fef587af3aeb66d2d0ca2 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav new file mode 100644 index 0000000000000000000000000000000000000000..3490fa05e85830ae95d0292e47e110b6d75365b5 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav new file mode 100644 index 0000000000000000000000000000000000000000..0543d3ccd5201cedc4af332862c3f442e1215985 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav new file mode 100644 index 0000000000000000000000000000000000000000..46d5125ff193249668b9bb20914b78c823a36c72 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav new file mode 100644 index 0000000000000000000000000000000000000000..c0bb47122dbb2c28303a740e3584244af200d622 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav new file mode 100644 index 0000000000000000000000000000000000000000..c7a8d3bf2066e0cc0fc8c7440a8fec973b0cd2fa Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav new file mode 100644 index 0000000000000000000000000000000000000000..b3c0e509bc20b4b23c3f7f0d0550f79ba3351ad8 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav new file mode 100644 index 0000000000000000000000000000000000000000..d1c2dd597285014176e15485b2825b63c140aaa4 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav new file mode 100644 index 0000000000000000000000000000000000000000..3475e43180ae8f832a333c0958652c5932670d40 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav new file mode 100644 index 0000000000000000000000000000000000000000..1db0c67769c101058e1de84cc7c3b477e07b8ba6 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav new file mode 100644 index 0000000000000000000000000000000000000000..93b821ad7a03abdb00b2e5775366c05a2c22680a Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav new file mode 100644 index 0000000000000000000000000000000000000000..31fbbae4eaeaf70dbeee317b73a1bdced70327c6 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav new file mode 100644 index 0000000000000000000000000000000000000000..12698f25952a38a0e085495b11e45983cd36487a --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4899e0da4615e883ad137c4ebecee7d753f2512e69b2610ee121046fafa228a2 +size 1282092 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_492597073098578f0605.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_492597073098578f0605.wav new file mode 100644 index 0000000000000000000000000000000000000000..ccb1e9aa6a218abf450e46c4b2704ae66156ec9d --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_492597073098578f0605.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:492597073098578f0605b080944b218a8c03fd49715aa37e5b6b9f2bbdc975fe +size 1282092 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav new file mode 100644 index 0000000000000000000000000000000000000000..918bb3e8699621f26aac61d6b287838166644597 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67d9409a306e3614ec3f84ea766046f8674e3cd4a8b13d1231dd6a356164f6d3 +size 1282092 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav new file mode 100644 index 0000000000000000000000000000000000000000..e6eb6195f70ba3a137cc86cebea511559be8a907 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c47ba927ac118ffaacca70183d37077fd7a6d6fc2c649e51b7fb5137609cd8e +size 1282092 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav new file mode 100644 index 0000000000000000000000000000000000000000..a481c7166ddf367e529f9e88c810bd758bb8f000 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89ad32d31f3e70178cc19b1fb1c8ceda0377b104575def497602c02ba8bd1196 +size 1282092 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav new file mode 100644 index 0000000000000000000000000000000000000000..e39eff166373d48c82cebfbb5a3b74256eb6e6c3 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd644667186ae0518a3c08763c4aabf1e1f4d5754f3025b1e40779e8f068d489 +size 1282092 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav new file mode 100644 index 0000000000000000000000000000000000000000..a49276f5ecb033eafcc70d9fdb0e85dccf3c98e2 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7405ef7b645b3265477b3e4e65bf1313dbc4445631e516872a054a694bb5e4a +size 1282092 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav new file mode 100644 index 0000000000000000000000000000000000000000..a968a955a3d0d652823f7d7c8d69381714266410 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav new file mode 100644 index 0000000000000000000000000000000000000000..932ba1880561b1dd05b265707de365ef5ce14822 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav new file mode 100644 index 0000000000000000000000000000000000000000..8b2db7ae423420de3b80a5a3147f104e26ef4503 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav new file mode 100644 index 0000000000000000000000000000000000000000..6b813b1a216d899bcc0fbfa7b2ae206f4e9e19aa Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav new file mode 100644 index 0000000000000000000000000000000000000000..77094fff1d5f304aace3d08a98f3d4a99f2ac7b6 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav new file mode 100644 index 0000000000000000000000000000000000000000..b9e5b2080b1a9e8c5088462ab00b737b641e04da Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav new file mode 100644 index 0000000000000000000000000000000000000000..6bbf1131e354d3af5af0c1cf82308485d1dcb9c7 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav new file mode 100644 index 0000000000000000000000000000000000000000..de391bf563f7f75771cad306b30df473b1f091d4 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav new file mode 100644 index 0000000000000000000000000000000000000000..486ffce0c41435231f18a203f99a40455f388d60 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav new file mode 100644 index 0000000000000000000000000000000000000000..a457b6444aaf098a14b8649a5cf6817cb9e1df7c Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav new file mode 100644 index 0000000000000000000000000000000000000000..627f0f90c3176420091fe2ba338b233cf1369db8 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav new file mode 100644 index 0000000000000000000000000000000000000000..7a82ba1fe870e4e485b7e34b2da48b04908f4a19 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav new file mode 100644 index 0000000000000000000000000000000000000000..ecbf6b7bbb40de79dedd971b0ae436f707412235 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav new file mode 100644 index 0000000000000000000000000000000000000000..106510561cb4f8c7ea4e8e305263d5f018bfb393 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav new file mode 100644 index 0000000000000000000000000000000000000000..e12fb4d882a15833374dd95924bdededb3613282 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09db3d4c95c46a3bae02e9caf38f6cc389d81d7d51882cc04228513ce4faa2e9 +size 1033260 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav new file mode 100644 index 0000000000000000000000000000000000000000..1c834a5ab5645a9b7535908cf6a87fa6d297eb83 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3349ee3cb71e3d789f004603adf9ae85b3ffccd3f0101888fb0a853e1955bfd2 +size 1033260 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav new file mode 100644 index 0000000000000000000000000000000000000000..6697fd32fc09ca0b27ef1312035f455a11248781 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63964662df5bc7731b7a2480a99cd41064e0649039c873203958da1818cd5c7f +size 1033260 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav new file mode 100644 index 0000000000000000000000000000000000000000..fcb431fad245c2bfee746cbf70a95c7ca2a6ac42 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ab4dededd2d3a052def8e7f92e274a8fc7ca7690dadf67dab7a600462b95ca1 +size 1033260 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav new file mode 100644 index 0000000000000000000000000000000000000000..7037da1a8b062e1fe503184fb5ac2fb43791d940 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8be5a72ea66619af0071ce4f2c06715203123028838719c5dd246b399d4dad4 +size 1033260 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav new file mode 100644 index 0000000000000000000000000000000000000000..f3752d7f30a6547737e6a799de18df10c373b2fc --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1c21fd700287f9f3109fcaf42061c1c993d84cb1ad0e31f77f23fdc8520183f +size 1033260 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav new file mode 100644 index 0000000000000000000000000000000000000000..9343fc74626f5680cbdae84147b0b6740132f3fe --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8feadb7228a5b136ce8392cd5c70ecb17bbec3f7b787ecd45858f7f5e9e45cd +size 1033260 diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav new file mode 100644 index 0000000000000000000000000000000000000000..48254ccf6712ec9b561f6c245e9f16ae946f8803 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav new file mode 100644 index 0000000000000000000000000000000000000000..20034e5bfb9004545ee35f3b9f4669cafabcbf9b Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_4871014c416be45744ef.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_4871014c416be45744ef.wav new file mode 100644 index 0000000000000000000000000000000000000000..107699a4efdae097d6ee77d3eb054c161b3a70a0 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_4871014c416be45744ef.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_819325490f684d367b39.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_819325490f684d367b39.wav new file mode 100644 index 0000000000000000000000000000000000000000..583ea73df9ca82386561d0b3c63f587c991257cd Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_819325490f684d367b39.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav new file mode 100644 index 0000000000000000000000000000000000000000..7014c4ba0a725fc9942682a2da1754661c8ff107 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav new file mode 100644 index 0000000000000000000000000000000000000000..17ed8d572865a599fda5c5af61e85a2ecbf6aa24 Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav new file mode 100644 index 0000000000000000000000000000000000000000..36d5839689b1590b463836c8450b80c6d3d281fd Binary files /dev/null and b/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav differ diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..60130745c2b20dcdb42fa2bf74ec09caed15b4f2 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json @@ -0,0 +1 @@ +{"columns": ["Target descriptions", "Target prompts", "Predicted transcriptions"], "data": [["A female speaker with a moderate pitch and slightly monotone tone delivers her words at a moderate speed in a confined environment with only slight background noise present.", "\"Well,\" said he, when Franz had concluded, \"what do you find to object to in all you have related?", " Well, said he, when France had concluded, What do you find to object to in all you have related?"], ["A female speaker with a slightly low-pitched, quite monotone voice delivers her words at a slightly faster than average pace in a confined environment with minimal background noise.", "The stop at Queenstown, the tedious passage up the Mersey, were things that he noted dimly through his growing impatience.", " The stop at Queen's Town, the tedious passage of the Mercy were things that he noted dimly through his growing impatience."], ["A female speaker with a slightly low-pitched voice delivers her words quite rapidly and with only slight monotone intonation in a confined, almost echo-free environment.", "'Yet I thought,' she said, 'that no man could have done this deed but Sigurd, Fafnir's bane, who was my true love.", " Yet I thought, she said, that no man could have done this deed, but Sigurd, Baffner's Bain, who was my true love."], ["Thomas speaks with clear and deliberate enunciation at a quite slow pace with high quality.", "Built with a lightning bolt? You got it! Sorry I couldn't provide more info! And Roland said, oh, he's so funny.", " belt with a lightning bolt. She got it. Sorry, I couldn't provide more in so."], ["Jerry speaks slightly slowly and with confusion in his voice with high quality.", "What's the latest from the All Pakistan Drama Page? How'd your shirt get torn, sweetie?", " What's the latest from the all-pissing trau-bide? Yeah, yeah, horned sweetie."], ["Jenny's voice is quite monotone and her speech is delivered very fast in a confined space with only slight ambient sound.", "I started to get involved as a Party Leader nearly a quarter of a century ago.", " I started to get involved as a party leader, nearly a quarter of a century ago."], ["A male voice with a low pitch and monotone tone speaks quite fast in a confined and clear recording environment.", "He started up nervously from the stone block for he could no longer quench the flame in his blood.", " He started up nervously from the stone block for he could no longer quench the flame in his blood."]]} \ No newline at end of file diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json new file mode 100644 index 0000000000000000000000000000000000000000..2d631cfd5837999bdb6587dccd085e8bd727805b --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json @@ -0,0 +1 @@ +{"columns": ["Target descriptions", "Target prompts", "Predicted transcriptions"], "data": [["A female speaker with a moderate pitch and slightly monotone tone delivers her words at a moderate speed in a confined environment with only slight background noise present.", "\"Well,\" said he, when Franz had concluded, \"what do you find to object to in all you have related?", " Well, said he, when Franz had concluded, What do you find to object to, in all you have related?"], ["A female speaker with a slightly low-pitched, quite monotone voice delivers her words at a slightly faster than average pace in a confined environment with minimal background noise.", "The stop at Queenstown, the tedious passage up the Mersey, were things that he noted dimly through his growing impatience.", " The stop at Queenstown, the tedious passage up the Mercy were things that he noted dimly through his growing impatience."], ["A female speaker with a slightly low-pitched voice delivers her words quite rapidly and with only slight monotone intonation in a confined, almost echo-free environment.", "'Yet I thought,' she said, 'that no man could have done this deed but Sigurd, Fafnir's bane, who was my true love.", " Yet I thought she said, that no man could have done the steed but Sigurd Faafner's Bain, who was my true love."], ["Thomas speaks with clear and deliberate enunciation at a quite slow pace with high quality.", "Built with a lightning bolt? You got it! Sorry I couldn't provide more info! And Roland said, oh, he's so funny.", " built with a lightning bolt. You got it. Sorry. I couldn't provide more info. And Roland said, oh, he's so funny."], ["Jerry speaks slightly slowly and with confusion in his voice with high quality.", "What's the latest from the All Pakistan Drama Page? How'd your shirt get torn, sweetie?", " What's the latest from the all-pastain drama page? How your shirt get torn, sweetie."], ["Jenny's voice is quite monotone and her speech is delivered very fast in a confined space with only slight ambient sound.", "I started to get involved as a Party Leader nearly a quarter of a century ago.", " I started to get involved as a party leader nearly a quarter of a century ago."], ["A male voice with a low pitch and monotone tone speaks quite fast in a confined and clear recording environment.", "He started up nervously from the stone block for he could no longer quench the flame in his blood.", " He started up nervously from the stone block, for he could no longer quench the flame in his blood."]]} \ No newline at end of file diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..778fca8c0bcb71920196651b2bfd9628803f6aa0 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json @@ -0,0 +1 @@ +{"columns": ["Target descriptions", "Target prompts", "Predicted transcriptions"], "data": [["A female speaker with a moderate pitch and slightly monotone tone delivers her words at a moderate speed in a confined environment with only slight background noise present.", "\"Well,\" said he, when Franz had concluded, \"what do you find to object to in all you have related?", " Well,\" said he, when France had concluded, what you will find to object to in all you have related?"], ["A female speaker with a slightly low-pitched, quite monotone voice delivers her words at a slightly faster than average pace in a confined environment with minimal background noise.", "The stop at Queenstown, the tedious passage up the Mersey, were things that he noted dimly through his growing impatience.", " The stop at Queenstown, the tedious passage of the Mercy were things that he noted dimly through his growing impatience."], ["A female speaker with a slightly low-pitched voice delivers her words quite rapidly and with only slight monotone intonation in a confined, almost echo-free environment.", "'Yet I thought,' she said, 'that no man could have done this deed but Sigurd, Fafnir's bane, who was my true love.", " Yet I thought she said, that no man could have done this deed but Sigurd Fafniz Bain, who was my true love."], ["Thomas speaks with clear and deliberate enunciation at a quite slow pace with high quality.", "Built with a lightning bolt? You got it! Sorry I couldn't provide more info! And Roland said, oh, he's so funny.", " Built with a lightning bolt, you got it! Sorry, I couldn't provide more inset."], ["Jerry speaks slightly slowly and with confusion in his voice with high quality.", "What's the latest from the All Pakistan Drama Page? How'd your shirt get torn, sweetie?", " What's the latest from the old rubber page and you should get torn sweetie"], ["Jenny's voice is quite monotone and her speech is delivered very fast in a confined space with only slight ambient sound.", "I started to get involved as a Party Leader nearly a quarter of a century ago.", " I started to get involved as a party leader nearly a quarter of a century ago."], ["A male voice with a low pitch and monotone tone speaks quite fast in a confined and clear recording environment.", "He started up nervously from the stone block for he could no longer quench the flame in his blood.", " He started up nervously from the stone block for he could no longer quench the flame in his blood."]]} \ No newline at end of file diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json new file mode 100644 index 0000000000000000000000000000000000000000..12cc08792bf2db34dbe8a3d558532487e539d862 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json @@ -0,0 +1 @@ +{"columns": ["Target descriptions", "Target prompts", "Predicted transcriptions"], "data": [["A female speaker with a moderate pitch and slightly monotone tone delivers her words at a moderate speed in a confined environment with only slight background noise present.", "\"Well,\" said he, when Franz had concluded, \"what do you find to object to in all you have related?", " Well, said he, when France had concluded, what do you find to object to, in all you have related?"], ["A female speaker with a slightly low-pitched, quite monotone voice delivers her words at a slightly faster than average pace in a confined environment with minimal background noise.", "The stop at Queenstown, the tedious passage up the Mersey, were things that he noted dimly through his growing impatience.", " At the stop at Queenstown, the tedious passage of the Mersey were things that he noted Dimleith through his growing impatience."], ["A female speaker with a slightly low-pitched voice delivers her words quite rapidly and with only slight monotone intonation in a confined, almost echo-free environment.", "'Yet I thought,' she said, 'that no man could have done this deed but Sigurd, Fafnir's bane, who was my true love.", " Yet I thought she said, that no man could have done this deed, but Sigurd Faafner's Bain, who was my true love."], ["Thomas speaks with clear and deliberate enunciation at a quite slow pace with high quality.", "Built with a lightning bolt? You got it! Sorry I couldn't provide more info! And Roland said, oh, he's so funny.", " Built with a lightning bolt, you got it. Sorry, I couldn't provide more an-o. And Roland said, oh, he's so funny."], ["Jerry speaks slightly slowly and with confusion in his voice with high quality.", "What's the latest from the All Pakistan Drama Page? How'd your shirt get torn, sweetie?", " What's the latest from the all-Paysam drama page? How'd you a shirt get torn?"], ["Jenny's voice is quite monotone and her speech is delivered very fast in a confined space with only slight ambient sound.", "I started to get involved as a Party Leader nearly a quarter of a century ago.", " I started to get involved as a party leader, nearly a quarter of a century ago."], ["A male voice with a low pitch and monotone tone speaks quite fast in a confined and clear recording environment.", "He started up nervously from the stone block for he could no longer quench the flame in his blood.", " He started up nervously from the stone block, for he could no longer quench the flame in his blood."]]} \ No newline at end of file diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..6c1206aecc081592c0d4d4ad3bb00ec6a4e3cb10 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json @@ -0,0 +1 @@ +{"columns": ["Target descriptions", "Target prompts", "Predicted transcriptions"], "data": [["A female speaker with a moderate pitch and slightly monotone tone delivers her words at a moderate speed in a confined environment with only slight background noise present.", "\"Well,\" said he, when Franz had concluded, \"what do you find to object to in all you have related?", " Well, said he, when Franz had concluded, what do you find to object to in all you have related?"], ["A female speaker with a slightly low-pitched, quite monotone voice delivers her words at a slightly faster than average pace in a confined environment with minimal background noise.", "The stop at Queenstown, the tedious passage up the Mersey, were things that he noted dimly through his growing impatience.", " The stupequins down a tedious passage of the Mercy, for things that he noted dimly through his growing impatience."], ["A female speaker with a slightly low-pitched voice delivers her words quite rapidly and with only slight monotone intonation in a confined, almost echo-free environment.", "'Yet I thought,' she said, 'that no man could have done this deed but Sigurd, Fafnir's bane, who was my true love.", " Yet I thought, she said, that no man could have done this deed but Sigurd Fafner's Bain, who was my true love."], ["Thomas speaks with clear and deliberate enunciation at a quite slow pace with high quality.", "Built with a lightning bolt? You got it! Sorry I couldn't provide more info! And Roland said, oh, he's so funny.", " Built with a lightning bolt. You got it. Sorry, I couldn't provide more info. And Roland said, oh, he's so funny."], ["Jerry speaks slightly slowly and with confusion in his voice with high quality.", "What's the latest from the All Pakistan Drama Page? How'd your shirt get torn, sweetie?", " What's the least from the all-passing ground you should get torn, sweet?"], ["Jenny's voice is quite monotone and her speech is delivered very fast in a confined space with only slight ambient sound.", "I started to get involved as a Party Leader nearly a quarter of a century ago.", " I started to get involved as a party leader nearly a quarter of a century ago."], ["A male voice with a low pitch and monotone tone speaks quite fast in a confined and clear recording environment.", "He started up nervously from the stone block for he could no longer quench the flame in his blood.", " He started up nervously from the stone block, for he could no longer quench the flame in his blood."]]} \ No newline at end of file diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..d888167c332f614ae55c77e0af2fb8ba2fc3d2de --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json @@ -0,0 +1 @@ +{"columns": ["Target descriptions", "Target prompts", "Predicted transcriptions"], "data": [["A female speaker with a moderate pitch and slightly monotone tone delivers her words at a moderate speed in a confined environment with only slight background noise present.", "\"Well,\" said he, when Franz had concluded, \"what do you find to object to in all you have related?", " Well, said he, when France had concluded, what do you find to object to in all you have related?"], ["A female speaker with a slightly low-pitched, quite monotone voice delivers her words at a slightly faster than average pace in a confined environment with minimal background noise.", "The stop at Queenstown, the tedious passage up the Mersey, were things that he noted dimly through his growing impatience.", " The stop at Queenstown, the tedious passage of the Mercy were things that he noted dimly through his growing impatience."], ["A female speaker with a slightly low-pitched voice delivers her words quite rapidly and with only slight monotone intonation in a confined, almost echo-free environment.", "'Yet I thought,' she said, 'that no man could have done this deed but Sigurd, Fafnir's bane, who was my true love.", " Yet I thought, she said, that no man could have done this d'eat but Sigurd Faftner's Bain, who was my true love."], ["Thomas speaks with clear and deliberate enunciation at a quite slow pace with high quality.", "Built with a lightning bolt? You got it! Sorry I couldn't provide more info! And Roland said, oh, he's so funny.", " Built with a lightning bolt. You got it. Sorry, I couldn't provide more in foe. And Roland said, oh, he's so funny."], ["Jerry speaks slightly slowly and with confusion in his voice with high quality.", "What's the latest from the All Pakistan Drama Page? How'd your shirt get torn, sweetie?", " What's the latest from the old page? How'd your shirt get torn, sweetie?"], ["Jenny's voice is quite monotone and her speech is delivered very fast in a confined space with only slight ambient sound.", "I started to get involved as a Party Leader nearly a quarter of a century ago.", " I started to get involved as a party leader nearly a quarter of a century ago."], ["A male voice with a low pitch and monotone tone speaks quite fast in a confined and clear recording environment.", "He started up nervously from the stone block for he could no longer quench the flame in his blood.", " He started up nervously from the stone block, for he could no longer quench the flame in his blood."]]} \ No newline at end of file diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..6d2b75b5a44ef7d48a5cd078758eda93e2e8abd6 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json @@ -0,0 +1 @@ +{"columns": ["Target descriptions", "Target prompts", "Predicted transcriptions"], "data": [["A female speaker with a moderate pitch and slightly monotone tone delivers her words at a moderate speed in a confined environment with only slight background noise present.", "\"Well,\" said he, when Franz had concluded, \"what do you find to object to in all you have related?", " Wow, said he, when France had concluded, What do you find your object to in all you have relayed?"], ["A female speaker with a slightly low-pitched, quite monotone voice delivers her words at a slightly faster than average pace in a confined environment with minimal background noise.", "The stop at Queenstown, the tedious passage up the Mersey, were things that he noted dimly through his growing impatience.", " The stop at Queenstown, the tedious passage up the Mersey were things that he noted dimly through his growing impatience."], ["A female speaker with a slightly low-pitched voice delivers her words quite rapidly and with only slight monotone intonation in a confined, almost echo-free environment.", "'Yet I thought,' she said, 'that no man could have done this deed but Sigurd, Fafnir's bane, who was my true love.", " Yet I thought she said, no man could have done this deed but Sigurd Faafner's Bain, who was my true love."], ["Thomas speaks with clear and deliberate enunciation at a quite slow pace with high quality.", "Built with a lightning bolt? You got it! Sorry I couldn't provide more info! And Roland said, oh, he's so funny.", " Built with a lightning bolt. You got it. Sorry, I couldn't provide more info. And Roland said, oh, he's so funny."], ["Jerry speaks slightly slowly and with confusion in his voice with high quality.", "What's the latest from the All Pakistan Drama Page? How'd your shirt get torn, sweetie?", " What's the latest from the all-peasing drama page? How'd your shirt get thored, sweetie?"], ["Jenny's voice is quite monotone and her speech is delivered very fast in a confined space with only slight ambient sound.", "I started to get involved as a Party Leader nearly a quarter of a century ago.", " I started to get involved as a party leader nearly a quarter of a century ago."], ["A male voice with a low pitch and monotone tone speaks quite fast in a confined and clear recording environment.", "He started up nervously from the stone block for he could no longer quench the flame in his blood.", " He started up nervously from the stone block, for he could no longer quench the flame in his blood."]]} \ No newline at end of file diff --git a/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json new file mode 100644 index 0000000000000000000000000000000000000000..1831973c31ddf581d3208fcbf50ab6ff344a23e3 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json @@ -0,0 +1 @@ +{"columns": ["Target descriptions", "Target prompts", "Predicted transcriptions"], "data": [["A female speaker with a moderate pitch and slightly monotone tone delivers her words at a moderate speed in a confined environment with only slight background noise present.", "\"Well,\" said he, when Franz had concluded, \"what do you find to object to in all you have related?", " Well, said he, when Franz had concluded, what do you find to object to in all you have related?"], ["A female speaker with a slightly low-pitched, quite monotone voice delivers her words at a slightly faster than average pace in a confined environment with minimal background noise.", "The stop at Queenstown, the tedious passage up the Mersey, were things that he noted dimly through his growing impatience.", " The stop at Queenstown, the tedious passage up the Mercy were things that he noted dimly through his growing impatience."], ["A female speaker with a slightly low-pitched voice delivers her words quite rapidly and with only slight monotone intonation in a confined, almost echo-free environment.", "'Yet I thought,' she said, 'that no man could have done this deed but Sigurd, Fafnir's bane, who was my true love.", " Yell thought she said that no man could have done this deed but Sigurd Faafner's Bain, who was my true love."], ["Thomas speaks with clear and deliberate enunciation at a quite slow pace with high quality.", "Built with a lightning bolt? You got it! Sorry I couldn't provide more info! And Roland said, oh, he's so funny.", " Built with a lightning bolt? You got it. Sorry I couldn't provide more in thunfo. And Brulin said, Oh, he's so funny."], ["Jerry speaks slightly slowly and with confusion in his voice with high quality.", "What's the latest from the All Pakistan Drama Page? How'd your shirt get torn, sweetie?", " What's the latest from the all-Paysane drama page? How'd your shirt get torn, sweetie?"], ["Jenny's voice is quite monotone and her speech is delivered very fast in a confined space with only slight ambient sound.", "I started to get involved as a Party Leader nearly a quarter of a century ago.", " I started to get involved as a party leader nearly a quarter of a century ago."], ["A male voice with a low pitch and monotone tone speaks quite fast in a confined and clear recording environment.", "He started up nervously from the stone block for he could no longer quench the flame in his blood.", " He started out nervously from the stone block, for he could no longer quench the flame in his blood."]]} \ No newline at end of file diff --git a/wandb/run-20240513_205249-qaoje1x9/files/output.log b/wandb/run-20240513_205249-qaoje1x9/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..21b73cc9688a11f79c30c5dfa0d364402ff5e591 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/output.log @@ -0,0 +1,8546 @@ +05/13/2024 20:52:52 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, 16-bits training: False +05/13/2024 20:52:52 - INFO - __main__ - Training/evaluation parameters ParlerTTSTrainingArguments( +_n_gpu=1, +accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, +adafactor=False, +adam_beta1=0.9, +adam_beta2=0.99, +adam_epsilon=1e-08, +audio_encoder_per_device_batch_size=4, +auto_find_batch_size=False, +batch_eval_metrics=False, +bf16=False, +bf16_full_eval=False, +data_seed=None, +dataloader_drop_last=False, +dataloader_num_workers=4, +dataloader_persistent_workers=False, +dataloader_pin_memory=True, +dataloader_prefetch_factor=None, +ddp_backend=None, +ddp_broadcast_buffers=None, +ddp_bucket_cap_mb=None, +ddp_find_unused_parameters=None, +ddp_timeout=1800, +debug=[], +deepspeed=None, +disable_tqdm=False, +dispatch_batches=None, +do_eval=True, +do_predict=False, +do_train=True, +dtype=bfloat16, +eval_accumulation_steps=None, +eval_delay=0, +eval_do_concat_batches=True, +eval_steps=None, +eval_strategy=IntervalStrategy.EPOCH, +evaluation_strategy=epoch, +fp16=False, +fp16_backend=auto, +fp16_full_eval=False, +fp16_opt_level=O1, +fsdp=[], +fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, +fsdp_min_num_params=0, +fsdp_transformer_layer_cls_to_wrap=None, +full_determinism=False, +generation_config=None, +generation_max_length=None, +generation_num_beams=None, +gradient_accumulation_steps=8, +gradient_checkpointing=True, +gradient_checkpointing_kwargs=None, +greater_is_better=None, +group_by_length=True, +half_precision_backend=auto, +hub_always_push=False, +hub_model_id=None, +hub_private_repo=False, +hub_strategy=HubStrategy.EVERY_SAVE, +hub_token=, +ignore_data_skip=False, +include_inputs_for_metrics=True, +include_num_input_tokens_seen=False, +include_tokens_per_second=False, +jit_mode_eval=False, +label_names=None, +label_smoothing_factor=0.0, +learning_rate=8e-05, +length_column_name=length, +load_best_model_at_end=False, +local_rank=0, +log_level=passive, +log_level_replica=warning, +log_on_each_node=True, +logging_dir=../output_dir_training_constant_concat/runs/May13_20-52-47_hf-dgx-01, +logging_first_step=False, +logging_nan_inf_filter=True, +logging_steps=5, +logging_strategy=IntervalStrategy.STEPS, +lr_scheduler_kwargs={}, +lr_scheduler_type=SchedulerType.COSINE, +max_grad_norm=1.0, +max_steps=-1, +metric_for_best_model=None, +mp_parameters=, +neftune_noise_alpha=None, +no_cuda=False, +num_train_epochs=8, +optim=OptimizerNames.ADAMW_TORCH, +optim_args=None, +optim_target_modules=None, +output_dir=../output_dir_training_constant_concat/, +overwrite_output_dir=True, +past_index=-1, +per_device_eval_batch_size=16, +per_device_train_batch_size=16, +predict_with_generate=True, +prediction_loss_only=False, +push_to_hub=False, +push_to_hub_model_id=None, +push_to_hub_organization=None, +push_to_hub_token=, +ray_scope=last, +remove_unused_columns=True, +report_to=['wandb'], +restore_callback_states_from_checkpoint=False, +resume_from_checkpoint=None, +run_name=../output_dir_training_constant_concat/, +save_on_each_node=False, +save_only_model=False, +save_safetensors=True, +save_steps=500, +save_strategy=IntervalStrategy.EPOCH, +save_total_limit=5, +seed=456, +skip_memory_metrics=True, +sortish_sampler=False, +split_batches=None, +tf32=None, +torch_compile=False, +torch_compile_backend=None, +torch_compile_mode=None, +torchdynamo=None, +tpu_metrics_debug=False, +tpu_num_cores=None, +use_cpu=False, +use_ipex=False, +use_legacy_prediction_loop=False, +use_mps_device=False, +warmup_ratio=0.0, +warmup_steps=250, +weight_decay=0.01, +) +05/13/2024 20:52:53 - WARNING - __main__ - Disabling fast tokenizer warning: https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py#L3231-L3235 +loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--parler-tts--dac_44khZ_8kbps/snapshots/db52bea859d9411e0beb44a3ea923a8731ee4197/preprocessor_config.json +Feature extractor EncodecFeatureExtractor { + "chunk_length_s": null, + "feature_extractor_type": "EncodecFeatureExtractor", + "feature_size": 1, + "overlap": null, + "padding_side": "right", + "padding_value": 0.0, + "return_attention_mask": true, + "sampling_rate": 44100 +} +loading file spiece.model from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/spiece.model +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/tokenizer.json +loading file added_tokens.json from cache at None +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/tokenizer_config.json +You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers +loading file spiece.model from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/spiece.model +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/tokenizer.json +loading file added_tokens.json from cache at None +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--parler-tts--parler_tts_mini_v0.1/snapshots/e02fd18e77d38b49a85c7a9a85189a64b8472544/tokenizer_config.json +Combining datasets...: 0%| | 0/4 [00:00": 50327, + "<|am|>": 50334, + "<|ar|>": 50272, + "<|as|>": 50350, + "<|az|>": 50304, + "<|ba|>": 50355, + "<|be|>": 50330, + "<|bg|>": 50292, + "<|bn|>": 50302, + "<|bo|>": 50347, + "<|br|>": 50309, + "<|bs|>": 50315, + "<|ca|>": 50270, + "<|cs|>": 50283, + "<|cy|>": 50297, + "<|da|>": 50285, + "<|de|>": 50261, + "<|el|>": 50281, + "<|en|>": 50259, + "<|es|>": 50262, + "<|et|>": 50307, + "<|eu|>": 50310, + "<|fa|>": 50300, + "<|fi|>": 50277, + "<|fo|>": 50338, + "<|fr|>": 50265, + "<|gl|>": 50319, + "<|gu|>": 50333, + "<|haw|>": 50352, + "<|ha|>": 50354, + "<|he|>": 50279, + "<|hi|>": 50276, + "<|hr|>": 50291, + "<|ht|>": 50339, + "<|hu|>": 50286, + "<|hy|>": 50312, + "<|id|>": 50275, + "<|is|>": 50311, + "<|it|>": 50274, + "<|ja|>": 50266, + "<|jw|>": 50356, + "<|ka|>": 50329, + "<|kk|>": 50316, + "<|km|>": 50323, + "<|kn|>": 50306, + "<|ko|>": 50264, + "<|la|>": 50294, + "<|lb|>": 50345, + "<|ln|>": 50353, + "<|lo|>": 50336, + "<|lt|>": 50293, + "<|lv|>": 50301, + "<|mg|>": 50349, + "<|mi|>": 50295, + "<|mk|>": 50308, + "<|ml|>": 50296, + "<|mn|>": 50314, + "<|mr|>": 50320, + "<|ms|>": 50282, + "<|mt|>": 50343, + "<|my|>": 50346, + "<|ne|>": 50313, + "<|nl|>": 50271, + "<|nn|>": 50342, + "<|no|>": 50288, + "<|oc|>": 50328, + "<|pa|>": 50321, + "<|pl|>": 50269, + "<|ps|>": 50340, + "<|pt|>": 50267, + "<|ro|>": 50284, + "<|ru|>": 50263, + "<|sa|>": 50344, + "<|sd|>": 50332, + "<|si|>": 50322, + "<|sk|>": 50298, + "<|sl|>": 50305, + "<|sn|>": 50324, + "<|so|>": 50326, + "<|sq|>": 50317, + "<|sr|>": 50303, + "<|su|>": 50357, + "<|sv|>": 50273, + "<|sw|>": 50318, + "<|ta|>": 50287, + "<|te|>": 50299, + "<|tg|>": 50331, + "<|th|>": 50289, + "<|tk|>": 50341, + "<|tl|>": 50348, + "<|tr|>": 50268, + "<|tt|>": 50351, + "<|uk|>": 50280, + "<|ur|>": 50290, + "<|uz|>": 50337, + "<|vi|>": 50278, + "<|yi|>": 50335, + "<|yo|>": 50325, + "<|zh|>": 50260 + }, + "language": "<|en|>", + "max_initial_timestamp_index": 50, + "max_length": 448, + "no_timestamps_token_id": 50363, + "pad_token_id": 50257, + "prev_sot_token_id": 50361, + "return_timestamps": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "task": "transcribe", + "task_to_id": { + "transcribe": 50359, + "translate": 50358 + }, + "use_scan": false +} +loading file vocab.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/vocab.json +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer.json +loading file merges.txt from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/merges.txt +loading file normalizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/normalizer.json +loading file added_tokens.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/added_tokens.json +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer_config.json +Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. +loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/preprocessor_config.json +Feature extractor WhisperFeatureExtractor { + "chunk_length": 30, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 80, + "hop_length": 160, + "n_fft": 400, + "n_samples": 480000, + "nb_max_frames": 3000, + "padding_side": "right", + "padding_value": 0.0, + "processor_class": "WhisperProcessor", + "return_attention_mask": false, + "sampling_rate": 16000 +} +Disabling tokenizer parallelism, we're using DataLoader multithreading already +You have passed task=transcribe, but also have set `forced_decoder_ids` to [[1, 50259], [2, 50359], [3, 50363]] which creates a conflict. `forced_decoder_ids` will be ignored in favor of task=transcribe. +Train steps ... : 12%|███████▌ | 127/1016 [14:35<1:36:38, 6.52s/it] + + +Train steps ... : 13%|███████▌ | 129/1016 [14:58<4:18:50, 17.51s/it] + + + + + +Train steps ... : 13%|███████▉ | 134/1016 [15:20<1:25:55, 5.85s/it] + + + + + +Train steps ... : 14%|████████▏ | 139/1016 [15:59<1:24:16, 5.77s/it] + + + + + +Train steps ... : 14%|████████▌ | 144/1016 [16:32<1:30:54, 6.26s/it] + + + + + + +Train steps ... : 15%|████████▊ | 150/1016 [17:11<1:39:05, 6.86s/it] + + + + +Train steps ... : 15%|█████████ | 154/1016 [17:36<1:57:20, 8.17s/it] + + + + + +Train steps ... : 16%|█████████▍ | 159/1016 [17:58<1:00:22, 4.23s/it] + + + + + +Train steps ... : 16%|█████████▋ | 164/1016 [18:38<1:18:31, 5.53s/it] + + + + + + +Train steps ... : 17%|██████████ | 170/1016 [19:16<1:18:19, 5.56s/it] + + + + + +Train steps ... : 17%|██████████▎ | 175/1016 [19:55<1:39:00, 7.06s/it] + + + +Train steps ... : 18%|██████████▌ | 179/1016 [20:22<1:59:21, 8.56s/it] + + + + + +Train steps ... : 18%|██████████▊ | 184/1016 [20:44<1:00:33, 4.37s/it] + + + + + +Train steps ... : 19%|███████████▏ | 189/1016 [21:21<1:14:01, 5.37s/it] + + + + + + +Train steps ... : 19%|███████████▌ | 195/1016 [22:03<1:19:53, 5.84s/it] + + + + +Train steps ... : 20%|███████████▊ | 199/1016 [22:35<1:46:07, 7.79s/it] + + + + + + +Train steps ... : 20%|████████████ | 205/1016 [23:19<2:05:46, 9.31s/it] + + + + +Train steps ... : 21%|████████████▎ | 209/1016 [23:34<1:03:01, 4.69s/it] + + + + + + +Train steps ... : 21%|████████████▋ | 215/1016 [24:18<1:03:48, 4.78s/it] + + + + +Train steps ... : 22%|████████████▉ | 219/1016 [24:56<1:35:06, 7.16s/it] + + + + + +Train steps ... : 22%|█████████████▏ | 224/1016 [25:31<1:44:27, 7.91s/it] + + + + + + +Train steps ... : 23%|█████████████▌ | 230/1016 [26:11<1:47:46, 8.23s/it] + + + + + +Train steps ... : 23%|█████████████▉ | 235/1016 [26:41<1:41:09, 7.77s/it] + + + + + +Train steps ... : 24%|██████████████▋ | 240/1016 [27:05<57:20, 4.43s/it] + + + + + +Train steps ... : 24%|██████████████▍ | 245/1016 [27:45<1:15:08, 5.85s/it] + + + + +Train steps ... : 25%|██████████████▋ | 249/1016 [28:19<1:46:53, 8.36s/it] + + + + + +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 2.07it/s] +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 2.07it/s] +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading weights file model.safetensors from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/model.safetensors +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "max_length": 448, + "pad_token_id": 50257, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ] +} +All model checkpoint weights were used when initializing WhisperForConditionalGeneration. +All the weights of WhisperForConditionalGeneration were initialized from the model checkpoint at distil-whisper/distil-large-v2. +If your task is similar to the task the model of the checkpoint was trained on, you can already use WhisperForConditionalGeneration for predictions without further training. +loading configuration file generation_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/generation_config.json +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "is_multilingual": true, + "lang_to_id": { + "<|af|>": 50327, + "<|am|>": 50334, + "<|ar|>": 50272, + "<|as|>": 50350, + "<|az|>": 50304, + "<|ba|>": 50355, + "<|be|>": 50330, + "<|bg|>": 50292, + "<|bn|>": 50302, + "<|bo|>": 50347, + "<|br|>": 50309, + "<|bs|>": 50315, + "<|ca|>": 50270, + "<|cs|>": 50283, + "<|cy|>": 50297, + "<|da|>": 50285, + "<|de|>": 50261, + "<|el|>": 50281, + "<|en|>": 50259, + "<|es|>": 50262, + "<|et|>": 50307, + "<|eu|>": 50310, + "<|fa|>": 50300, + "<|fi|>": 50277, + "<|fo|>": 50338, + "<|fr|>": 50265, + "<|gl|>": 50319, + "<|gu|>": 50333, + "<|haw|>": 50352, + "<|ha|>": 50354, + "<|he|>": 50279, + "<|hi|>": 50276, + "<|hr|>": 50291, + "<|ht|>": 50339, + "<|hu|>": 50286, + "<|hy|>": 50312, + "<|id|>": 50275, + "<|is|>": 50311, + "<|it|>": 50274, + "<|ja|>": 50266, + "<|jw|>": 50356, + "<|ka|>": 50329, + "<|kk|>": 50316, + "<|km|>": 50323, + "<|kn|>": 50306, + "<|ko|>": 50264, + "<|la|>": 50294, + "<|lb|>": 50345, + "<|ln|>": 50353, + "<|lo|>": 50336, + "<|lt|>": 50293, + "<|lv|>": 50301, + "<|mg|>": 50349, + "<|mi|>": 50295, + "<|mk|>": 50308, + "<|ml|>": 50296, + "<|mn|>": 50314, + "<|mr|>": 50320, + "<|ms|>": 50282, + "<|mt|>": 50343, + "<|my|>": 50346, + "<|ne|>": 50313, + "<|nl|>": 50271, + "<|nn|>": 50342, + "<|no|>": 50288, + "<|oc|>": 50328, + "<|pa|>": 50321, + "<|pl|>": 50269, + "<|ps|>": 50340, + "<|pt|>": 50267, + "<|ro|>": 50284, + "<|ru|>": 50263, + "<|sa|>": 50344, + "<|sd|>": 50332, + "<|si|>": 50322, + "<|sk|>": 50298, + "<|sl|>": 50305, + "<|sn|>": 50324, + "<|so|>": 50326, + "<|sq|>": 50317, + "<|sr|>": 50303, + "<|su|>": 50357, + "<|sv|>": 50273, + "<|sw|>": 50318, + "<|ta|>": 50287, + "<|te|>": 50299, + "<|tg|>": 50331, + "<|th|>": 50289, + "<|tk|>": 50341, + "<|tl|>": 50348, + "<|tr|>": 50268, + "<|tt|>": 50351, + "<|uk|>": 50280, + "<|ur|>": 50290, + "<|uz|>": 50337, + "<|vi|>": 50278, + "<|yi|>": 50335, + "<|yo|>": 50325, + "<|zh|>": 50260 + }, + "language": "<|en|>", + "max_initial_timestamp_index": 50, + "max_length": 448, + "no_timestamps_token_id": 50363, + "pad_token_id": 50257, + "prev_sot_token_id": 50361, + "return_timestamps": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "task": "transcribe", + "task_to_id": { + "transcribe": 50359, + "translate": 50358 + }, + "use_scan": false +} +loading file vocab.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/vocab.json +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer.json +loading file merges.txt from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/merges.txt +loading file normalizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/normalizer.json +loading file added_tokens.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/added_tokens.json +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer_config.json +Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. +loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/preprocessor_config.json +Feature extractor WhisperFeatureExtractor { + "chunk_length": 30, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 80, + "hop_length": 160, + "n_fft": 400, + "n_samples": 480000, + "nb_max_frames": 3000, + "padding_side": "right", + "padding_value": 0.0, + "processor_class": "WhisperProcessor", + "return_attention_mask": false, + "sampling_rate": 16000 +} +You have passed task=transcribe, but also have set `forced_decoder_ids` to [[1, 50259], [2, 50359], [3, 50363]] which creates a conflict. `forced_decoder_ids` will be ignored in favor of task=transcribe. +Train steps ... : 25%|███████████████ | 254/1016 [29:34<2:06:12, 9.94s/it] +Eval results for step (254 / 1016 | Eval Loss: 4.225829124450684 | Eval clap: 0.3507245182991028 | Eval wer: 24.242424242424242 |) + + + + + +Train steps ... : 25%|███████████████▎ | 259/1016 [30:17<2:26:23, 11.60s/it] + + + + + +Train steps ... : 26%|███████████████▌ | 264/1016 [30:53<1:58:19, 9.44s/it] + + + + + +Train steps ... : 26%|███████████████▉ | 269/1016 [31:24<1:37:22, 7.82s/it] + + + + + +Train steps ... : 27%|████████████████▋ | 274/1016 [31:52<59:57, 4.85s/it] + + + + + + +Train steps ... : 28%|█████████████████ | 280/1016 [32:31<58:51, 4.80s/it] + + + + + +Train steps ... : 28%|████████████████▊ | 285/1016 [33:09<1:17:57, 6.40s/it] + + + + +Train steps ... : 28%|█████████████████ | 289/1016 [33:38<1:37:44, 8.07s/it] + + + + + +Train steps ... : 29%|█████████████████▎ | 294/1016 [34:08<1:29:07, 7.41s/it] + + + + + +Train steps ... : 29%|█████████████████▋ | 299/1016 [34:38<1:00:44, 5.08s/it] + + + + + +Train steps ... : 30%|█████████████████▉ | 304/1016 [35:19<1:16:42, 6.46s/it] + + + + + + +Train steps ... : 31%|██████████████████▎ | 310/1016 [36:03<1:20:45, 6.86s/it] + + + + + +Train steps ... : 31%|██████████████████▌ | 315/1016 [36:39<1:28:43, 7.59s/it] + + + + + +Train steps ... : 31%|██████████████████▉ | 320/1016 [37:11<1:30:33, 7.81s/it] + + + + + +Train steps ... : 32%|███████████████████▏ | 325/1016 [37:37<1:10:13, 6.10s/it] + + + + +Train steps ... : 32%|███████████████████▍ | 329/1016 [38:05<1:07:33, 5.90s/it] + + + + + +Train steps ... : 33%|███████████████████▋ | 334/1016 [38:39<1:20:17, 7.06s/it] + + + + + + +Train steps ... : 33%|████████████████████ | 340/1016 [39:19<1:22:19, 7.31s/it] + + + + + +Train steps ... : 34%|████████████████████▎ | 345/1016 [39:51<1:28:56, 7.95s/it] + + + + + +Train steps ... : 34%|████████████████████▋ | 350/1016 [40:17<1:07:17, 6.06s/it] + + + + +Train steps ... : 35%|████████████████████▉ | 354/1016 [40:45<1:06:38, 6.04s/it] + + + + + + +Train steps ... : 35%|█████████████████████▎ | 360/1016 [41:25<1:08:25, 6.26s/it] + + + + + +Train steps ... : 36%|█████████████████████▌ | 365/1016 [41:59<1:17:28, 7.14s/it] + + + + +Train steps ... : 36%|█████████████████████▊ | 369/1016 [42:21<1:16:36, 7.10s/it] + + + + + +Train steps ... : 37%|██████████████████████▊ | 374/1016 [42:49<50:41, 4.74s/it] + + + + + + +Train steps ... : 37%|███████████████████████▏ | 380/1016 [43:31<54:18, 5.12s/it] + +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.37s/it] +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.37s/it] + +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading weights file model.safetensors from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/model.safetensors +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "max_length": 448, + "pad_token_id": 50257, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ] +} +All model checkpoint weights were used when initializing WhisperForConditionalGeneration. +All the weights of WhisperForConditionalGeneration were initialized from the model checkpoint at distil-whisper/distil-large-v2. +If your task is similar to the task the model of the checkpoint was trained on, you can already use WhisperForConditionalGeneration for predictions without further training. +loading configuration file generation_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/generation_config.json +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "is_multilingual": true, + "lang_to_id": { + "<|af|>": 50327, + "<|am|>": 50334, + "<|ar|>": 50272, + "<|as|>": 50350, + "<|az|>": 50304, + "<|ba|>": 50355, + "<|be|>": 50330, + "<|bg|>": 50292, + "<|bn|>": 50302, + "<|bo|>": 50347, + "<|br|>": 50309, + "<|bs|>": 50315, + "<|ca|>": 50270, + "<|cs|>": 50283, + "<|cy|>": 50297, + "<|da|>": 50285, + "<|de|>": 50261, + "<|el|>": 50281, + "<|en|>": 50259, + "<|es|>": 50262, + "<|et|>": 50307, + "<|eu|>": 50310, + "<|fa|>": 50300, + "<|fi|>": 50277, + "<|fo|>": 50338, + "<|fr|>": 50265, + "<|gl|>": 50319, + "<|gu|>": 50333, + "<|haw|>": 50352, + "<|ha|>": 50354, + "<|he|>": 50279, + "<|hi|>": 50276, + "<|hr|>": 50291, + "<|ht|>": 50339, + "<|hu|>": 50286, + "<|hy|>": 50312, + "<|id|>": 50275, + "<|is|>": 50311, + "<|it|>": 50274, + "<|ja|>": 50266, + "<|jw|>": 50356, + "<|ka|>": 50329, + "<|kk|>": 50316, + "<|km|>": 50323, + "<|kn|>": 50306, + "<|ko|>": 50264, + "<|la|>": 50294, + "<|lb|>": 50345, + "<|ln|>": 50353, + "<|lo|>": 50336, + "<|lt|>": 50293, + "<|lv|>": 50301, + "<|mg|>": 50349, + "<|mi|>": 50295, + "<|mk|>": 50308, + "<|ml|>": 50296, + "<|mn|>": 50314, + "<|mr|>": 50320, + "<|ms|>": 50282, + "<|mt|>": 50343, + "<|my|>": 50346, + "<|ne|>": 50313, + "<|nl|>": 50271, + "<|nn|>": 50342, + "<|no|>": 50288, + "<|oc|>": 50328, + "<|pa|>": 50321, + "<|pl|>": 50269, + "<|ps|>": 50340, + "<|pt|>": 50267, + "<|ro|>": 50284, + "<|ru|>": 50263, + "<|sa|>": 50344, + "<|sd|>": 50332, + "<|si|>": 50322, + "<|sk|>": 50298, + "<|sl|>": 50305, + "<|sn|>": 50324, + "<|so|>": 50326, + "<|sq|>": 50317, + "<|sr|>": 50303, + "<|su|>": 50357, + "<|sv|>": 50273, + "<|sw|>": 50318, + "<|ta|>": 50287, + "<|te|>": 50299, + "<|tg|>": 50331, + "<|th|>": 50289, + "<|tk|>": 50341, + "<|tl|>": 50348, + "<|tr|>": 50268, + "<|tt|>": 50351, + "<|uk|>": 50280, + "<|ur|>": 50290, + "<|uz|>": 50337, + "<|vi|>": 50278, + "<|yi|>": 50335, + "<|yo|>": 50325, + "<|zh|>": 50260 + }, + "language": "<|en|>", + "max_initial_timestamp_index": 50, + "max_length": 448, + "no_timestamps_token_id": 50363, + "pad_token_id": 50257, + "prev_sot_token_id": 50361, + "return_timestamps": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "task": "transcribe", + "task_to_id": { + "transcribe": 50359, + "translate": 50358 + }, + "use_scan": false +} +loading file vocab.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/vocab.json +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer.json +loading file merges.txt from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/merges.txt +loading file normalizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/normalizer.json +loading file added_tokens.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/added_tokens.json +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer_config.json +Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. +loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/preprocessor_config.json +Feature extractor WhisperFeatureExtractor { + "chunk_length": 30, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 80, + "hop_length": 160, + "n_fft": 400, + "n_samples": 480000, + "nb_max_frames": 3000, + "padding_side": "right", + "padding_value": 0.0, + "processor_class": "WhisperProcessor", + "return_attention_mask": false, + "sampling_rate": 16000 +} +You have passed task=transcribe, but also have set `forced_decoder_ids` to [[1, 50259], [2, 50359], [3, 50363]] which creates a conflict. `forced_decoder_ids` will be ignored in favor of task=transcribe. +Train steps ... : 38%|███████████████████████▎ | 381/1016 [44:21<43:55, 4.15s/it] + + + + +Train steps ... : 38%|██████████████████████▋ | 385/1016 [45:03<2:34:41, 14.71s/it] + + + + + +Train steps ... : 38%|███████████████████████▊ | 390/1016 [45:25<56:34, 5.42s/it] + + + + +Train steps ... : 39%|███████████████████████▎ | 394/1016 [46:01<1:12:00, 6.95s/it] + + + + + +Train steps ... : 39%|███████████████████████▌ | 399/1016 [46:37<1:19:37, 7.74s/it] + + + + + + +Train steps ... : 40%|███████████████████████▉ | 405/1016 [47:13<1:14:03, 7.27s/it] + + + + +Train steps ... : 40%|████████████████████████▉ | 409/1016 [47:26<39:20, 3.89s/it] + + + + + + +Train steps ... : 41%|█████████████████████████▎ | 415/1016 [48:07<43:33, 4.35s/it] + + + + + +Train steps ... : 41%|█████████████████████████▋ | 420/1016 [48:45<52:58, 5.33s/it] + + + + +Train steps ... : 42%|█████████████████████████ | 424/1016 [49:15<1:10:59, 7.20s/it] + + + + + +Train steps ... : 42%|█████████████████████████▎ | 429/1016 [49:48<1:20:51, 8.27s/it] + + + + + + +Train steps ... : 43%|█████████████████████████▋ | 435/1016 [50:29<1:24:59, 8.78s/it] + + + + +Train steps ... : 43%|██████████████████████████▊ | 439/1016 [50:49<50:57, 5.30s/it] + + + + + +Train steps ... : 44%|██████████████████████████▏ | 444/1016 [51:27<1:03:32, 6.67s/it] + + + + + + +Train steps ... : 44%|██████████████████████████▌ | 450/1016 [52:07<1:04:38, 6.85s/it] + + + + + +Train steps ... : 45%|██████████████████████████▊ | 455/1016 [52:43<1:14:15, 7.94s/it] + + + + + +Train steps ... : 45%|███████████████████████████▏ | 460/1016 [53:13<1:14:20, 8.02s/it] + + + + +Train steps ... : 46%|████████████████████████████▎ | 464/1016 [53:34<49:08, 5.34s/it] + + + + + + +Train steps ... : 46%|████████████████████████████▋ | 470/1016 [54:13<48:53, 5.37s/it] + + + + +Train steps ... : 47%|███████████████████████████▉ | 474/1016 [54:44<1:04:20, 7.12s/it] + + + + + + +Train steps ... : 47%|████████████████████████████▎ | 480/1016 [55:23<1:10:43, 7.92s/it] + + + + + +Train steps ... : 48%|████████████████████████████▋ | 485/1016 [55:55<1:14:41, 8.44s/it] + + + + +Train steps ... : 48%|█████████████████████████████▊ | 489/1016 [56:15<45:15, 5.15s/it] + + + + + + +Train steps ... : 49%|██████████████████████████████▏ | 495/1016 [56:55<46:58, 5.41s/it] + + + + + +Train steps ... : 49%|██████████████████████████████▌ | 500/1016 [57:31<56:07, 6.53s/it] + + + + +Train steps ... : 50%|█████████████████████████████▊ | 504/1016 [57:57<1:05:53, 7.72s/it] + + + + +Train steps ... : 50%|███████████████████████████████ | 508/1016 [58:15<40:38, 4.80s/it] +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.34s/it] + +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading weights file model.safetensors from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/model.safetensors +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "max_length": 448, + "pad_token_id": 50257, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ] +} +All model checkpoint weights were used when initializing WhisperForConditionalGeneration. +All the weights of WhisperForConditionalGeneration were initialized from the model checkpoint at distil-whisper/distil-large-v2. +If your task is similar to the task the model of the checkpoint was trained on, you can already use WhisperForConditionalGeneration for predictions without further training. +loading configuration file generation_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/generation_config.json +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "is_multilingual": true, + "lang_to_id": { + "<|af|>": 50327, + "<|am|>": 50334, + "<|ar|>": 50272, + "<|as|>": 50350, + "<|az|>": 50304, + "<|ba|>": 50355, + "<|be|>": 50330, + "<|bg|>": 50292, + "<|bn|>": 50302, + "<|bo|>": 50347, + "<|br|>": 50309, + "<|bs|>": 50315, + "<|ca|>": 50270, + "<|cs|>": 50283, + "<|cy|>": 50297, + "<|da|>": 50285, + "<|de|>": 50261, + "<|el|>": 50281, + "<|en|>": 50259, + "<|es|>": 50262, + "<|et|>": 50307, + "<|eu|>": 50310, + "<|fa|>": 50300, + "<|fi|>": 50277, + "<|fo|>": 50338, + "<|fr|>": 50265, + "<|gl|>": 50319, + "<|gu|>": 50333, + "<|haw|>": 50352, + "<|ha|>": 50354, + "<|he|>": 50279, + "<|hi|>": 50276, + "<|hr|>": 50291, + "<|ht|>": 50339, + "<|hu|>": 50286, + "<|hy|>": 50312, + "<|id|>": 50275, + "<|is|>": 50311, + "<|it|>": 50274, + "<|ja|>": 50266, + "<|jw|>": 50356, + "<|ka|>": 50329, + "<|kk|>": 50316, + "<|km|>": 50323, + "<|kn|>": 50306, + "<|ko|>": 50264, + "<|la|>": 50294, + "<|lb|>": 50345, + "<|ln|>": 50353, + "<|lo|>": 50336, + "<|lt|>": 50293, + "<|lv|>": 50301, + "<|mg|>": 50349, + "<|mi|>": 50295, + "<|mk|>": 50308, + "<|ml|>": 50296, + "<|mn|>": 50314, + "<|mr|>": 50320, + "<|ms|>": 50282, + "<|mt|>": 50343, + "<|my|>": 50346, + "<|ne|>": 50313, + "<|nl|>": 50271, + "<|nn|>": 50342, + "<|no|>": 50288, + "<|oc|>": 50328, + "<|pa|>": 50321, + "<|pl|>": 50269, + "<|ps|>": 50340, + "<|pt|>": 50267, + "<|ro|>": 50284, + "<|ru|>": 50263, + "<|sa|>": 50344, + "<|sd|>": 50332, + "<|si|>": 50322, + "<|sk|>": 50298, + "<|sl|>": 50305, + "<|sn|>": 50324, + "<|so|>": 50326, + "<|sq|>": 50317, + "<|sr|>": 50303, + "<|su|>": 50357, + "<|sv|>": 50273, + "<|sw|>": 50318, + "<|ta|>": 50287, + "<|te|>": 50299, + "<|tg|>": 50331, + "<|th|>": 50289, + "<|tk|>": 50341, + "<|tl|>": 50348, + "<|tr|>": 50268, + "<|tt|>": 50351, + "<|uk|>": 50280, + "<|ur|>": 50290, + "<|uz|>": 50337, + "<|vi|>": 50278, + "<|yi|>": 50335, + "<|yo|>": 50325, + "<|zh|>": 50260 + }, + "language": "<|en|>", + "max_initial_timestamp_index": 50, + "max_length": 448, + "no_timestamps_token_id": 50363, + "pad_token_id": 50257, + "prev_sot_token_id": 50361, + "return_timestamps": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "task": "transcribe", + "task_to_id": { + "transcribe": 50359, + "translate": 50358 + }, + "use_scan": false +} +loading file vocab.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/vocab.json +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer.json +loading file merges.txt from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/merges.txt +loading file normalizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/normalizer.json +loading file added_tokens.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/added_tokens.json +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer_config.json +Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. +loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/preprocessor_config.json +Feature extractor WhisperFeatureExtractor { + "chunk_length": 30, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 80, + "hop_length": 160, + "n_fft": 400, + "n_samples": 480000, + "nb_max_frames": 3000, + "padding_side": "right", + "padding_value": 0.0, + "processor_class": "WhisperProcessor", + "return_attention_mask": false, + "sampling_rate": 16000 +} +You have passed task=transcribe, but also have set `forced_decoder_ids` to [[1, 50259], [2, 50359], [3, 50363]] which creates a conflict. `forced_decoder_ids` will be ignored in favor of task=transcribe. +Eval results for step (508 / 1016 | Eval Loss: 4.146361351013184 | Eval clap: 0.402365118265152 | Eval wer: 21.96969696969697 |) + +Train steps ... : 50%|██████████████████████████████ | 510/1016 [59:13<2:13:02, 15.78s/it] + + + + +Train steps ... : 51%|██████████████████████████████▎ | 514/1016 [59:47<1:33:34, 11.18s/it] + + + + + +Train steps ... : 51%|█████████████████████████████▋ | 519/1016 [1:00:19<1:11:38, 8.65s/it] + + + + + + +Train steps ... : 52%|███████████████████████████████ | 525/1016 [1:00:57<59:30, 7.27s/it] + + + + + +Train steps ... : 52%|███████████████████████████████▎ | 530/1016 [1:01:24<37:17, 4.60s/it] + + + + +Train steps ... : 53%|███████████████████████████████▌ | 534/1016 [1:01:58<56:18, 7.01s/it] + + + + + + +Train steps ... : 53%|███████████████████████████████▉ | 540/1016 [1:02:40<59:56, 7.56s/it] + + + + + +Train steps ... : 54%|███████████████████████████████ | 545/1016 [1:03:12<1:01:43, 7.86s/it] + + + + + +Train steps ... : 54%|████████████████████████████████▍ | 550/1016 [1:03:41<56:37, 7.29s/it] + + + + +Train steps ... : 55%|████████████████████████████████▋ | 554/1016 [1:04:06<43:50, 5.69s/it] + + + + + + +Train steps ... : 55%|█████████████████████████████████ | 560/1016 [1:04:46<44:20, 5.83s/it] + + + + +Train steps ... : 56%|█████████████████████████████████▎ | 564/1016 [1:05:15<58:19, 7.74s/it] + + + + + +Train steps ... : 56%|█████████████████████████████████▌ | 569/1016 [1:05:44<56:09, 7.54s/it] + + + + + +Train steps ... : 56%|█████████████████████████████████▉ | 574/1016 [1:06:10<33:09, 4.50s/it] + + + + + + +Train steps ... : 57%|██████████████████████████████████▎ | 580/1016 [1:06:52<34:37, 4.77s/it] + + + + + +Train steps ... : 58%|██████████████████████████████████▌ | 585/1016 [1:07:29<42:24, 5.90s/it] + + + + +Train steps ... : 58%|██████████████████████████████████▊ | 589/1016 [1:07:59<57:48, 8.12s/it] + + + + + + +Train steps ... : 59%|███████████████████████████████████▏ | 595/1016 [1:08:37<55:16, 7.88s/it] + + + + + +Train steps ... : 59%|███████████████████████████████████▍ | 600/1016 [1:09:05<48:00, 6.92s/it] + + + + +Train steps ... : 59%|███████████████████████████████████▋ | 604/1016 [1:09:28<37:23, 5.44s/it] + + + + + + +Train steps ... : 60%|████████████████████████████████████ | 610/1016 [1:10:07<38:09, 5.64s/it] + + + + + +Train steps ... : 61%|████████████████████████████████████▎ | 615/1016 [1:10:46<51:22, 7.69s/it] + + + + + +Train steps ... : 61%|████████████████████████████████████▌ | 620/1016 [1:11:20<54:26, 8.25s/it] + + + + + +Train steps ... : 62%|████████████████████████████████████▉ | 625/1016 [1:11:47<44:58, 6.90s/it] + + + + +Train steps ... : 62%|█████████████████████████████████████▏ | 629/1016 [1:12:12<36:14, 5.62s/it] + + + + + + +Train steps ... : 62%|█████████████████████████████████████▌ | 635/1016 [1:12:52<37:02, 5.83s/it] +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.35s/it] +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.35s/it] + +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading weights file model.safetensors from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/model.safetensors +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "max_length": 448, + "pad_token_id": 50257, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ] +} +All model checkpoint weights were used when initializing WhisperForConditionalGeneration. +All the weights of WhisperForConditionalGeneration were initialized from the model checkpoint at distil-whisper/distil-large-v2. +If your task is similar to the task the model of the checkpoint was trained on, you can already use WhisperForConditionalGeneration for predictions without further training. +loading configuration file generation_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/generation_config.json +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "is_multilingual": true, + "lang_to_id": { + "<|af|>": 50327, + "<|am|>": 50334, + "<|ar|>": 50272, + "<|as|>": 50350, + "<|az|>": 50304, + "<|ba|>": 50355, + "<|be|>": 50330, + "<|bg|>": 50292, + "<|bn|>": 50302, + "<|bo|>": 50347, + "<|br|>": 50309, + "<|bs|>": 50315, + "<|ca|>": 50270, + "<|cs|>": 50283, + "<|cy|>": 50297, + "<|da|>": 50285, + "<|de|>": 50261, + "<|el|>": 50281, + "<|en|>": 50259, + "<|es|>": 50262, + "<|et|>": 50307, + "<|eu|>": 50310, + "<|fa|>": 50300, + "<|fi|>": 50277, + "<|fo|>": 50338, + "<|fr|>": 50265, + "<|gl|>": 50319, + "<|gu|>": 50333, + "<|haw|>": 50352, + "<|ha|>": 50354, + "<|he|>": 50279, + "<|hi|>": 50276, + "<|hr|>": 50291, + "<|ht|>": 50339, + "<|hu|>": 50286, + "<|hy|>": 50312, + "<|id|>": 50275, + "<|is|>": 50311, + "<|it|>": 50274, + "<|ja|>": 50266, + "<|jw|>": 50356, + "<|ka|>": 50329, + "<|kk|>": 50316, + "<|km|>": 50323, + "<|kn|>": 50306, + "<|ko|>": 50264, + "<|la|>": 50294, + "<|lb|>": 50345, + "<|ln|>": 50353, + "<|lo|>": 50336, + "<|lt|>": 50293, + "<|lv|>": 50301, + "<|mg|>": 50349, + "<|mi|>": 50295, + "<|mk|>": 50308, + "<|ml|>": 50296, + "<|mn|>": 50314, + "<|mr|>": 50320, + "<|ms|>": 50282, + "<|mt|>": 50343, + "<|my|>": 50346, + "<|ne|>": 50313, + "<|nl|>": 50271, + "<|nn|>": 50342, + "<|no|>": 50288, + "<|oc|>": 50328, + "<|pa|>": 50321, + "<|pl|>": 50269, + "<|ps|>": 50340, + "<|pt|>": 50267, + "<|ro|>": 50284, + "<|ru|>": 50263, + "<|sa|>": 50344, + "<|sd|>": 50332, + "<|si|>": 50322, + "<|sk|>": 50298, + "<|sl|>": 50305, + "<|sn|>": 50324, + "<|so|>": 50326, + "<|sq|>": 50317, + "<|sr|>": 50303, + "<|su|>": 50357, + "<|sv|>": 50273, + "<|sw|>": 50318, + "<|ta|>": 50287, + "<|te|>": 50299, + "<|tg|>": 50331, + "<|th|>": 50289, + "<|tk|>": 50341, + "<|tl|>": 50348, + "<|tr|>": 50268, + "<|tt|>": 50351, + "<|uk|>": 50280, + "<|ur|>": 50290, + "<|uz|>": 50337, + "<|vi|>": 50278, + "<|yi|>": 50335, + "<|yo|>": 50325, + "<|zh|>": 50260 + }, + "language": "<|en|>", + "max_initial_timestamp_index": 50, + "max_length": 448, + "no_timestamps_token_id": 50363, + "pad_token_id": 50257, + "prev_sot_token_id": 50361, + "return_timestamps": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "task": "transcribe", + "task_to_id": { + "transcribe": 50359, + "translate": 50358 + }, + "use_scan": false +} +loading file vocab.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/vocab.json +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer.json +loading file merges.txt from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/merges.txt +loading file normalizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/normalizer.json +loading file added_tokens.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/added_tokens.json +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer_config.json +Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. +loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/preprocessor_config.json +Feature extractor WhisperFeatureExtractor { + "chunk_length": 30, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 80, + "hop_length": 160, + "n_fft": 400, + "n_samples": 480000, + "nb_max_frames": 3000, + "padding_side": "right", + "padding_value": 0.0, + "processor_class": "WhisperProcessor", + "return_attention_mask": false, + "sampling_rate": 16000 +} +You have passed task=transcribe, but also have set `forced_decoder_ids` to [[1, 50259], [2, 50359], [3, 50363]] which creates a conflict. `forced_decoder_ids` will be ignored in favor of task=transcribe. +Eval results for step (635 / 1016 | Eval Loss: 4.128090858459473 | Eval clap: 0.3640345335006714 | Eval wer: 15.909090909090908 |) + + + + +Train steps ... : 63%|████████████████████████████████████▍ | 639/1016 [1:13:57<1:05:41, 10.46s/it] + + + + + +Train steps ... : 63%|██████████████████████████████████████ | 644/1016 [1:14:38<47:35, 7.68s/it] + + + + + +Train steps ... : 64%|██████████████████████████████████████▎ | 649/1016 [1:15:13<45:57, 7.51s/it] + + + + + + +Train steps ... : 64%|██████████████████████████████████████▋ | 655/1016 [1:15:53<45:59, 7.64s/it] + + + + +Train steps ... : 65%|██████████████████████████████████████▉ | 659/1016 [1:16:13<36:59, 6.22s/it] + + + + + +Train steps ... : 65%|███████████████████████████████████████▏ | 664/1016 [1:16:45<29:53, 5.09s/it] + + + + + +Train steps ... : 66%|███████████████████████████████████████▌ | 669/1016 [1:17:22<36:29, 6.31s/it] + + + + + +Train steps ... : 66%|███████████████████████████████████████▊ | 674/1016 [1:17:58<42:46, 7.50s/it] + + + + + +Train steps ... : 67%|████████████████████████████████████████ | 679/1016 [1:18:29<42:09, 7.51s/it] + + + + + + +Train steps ... : 67%|████████████████████████████████████████▍ | 685/1016 [1:19:07<44:01, 7.98s/it] + + + + +Train steps ... : 68%|████████████████████████████████████████▋ | 689/1016 [1:19:25<26:19, 4.83s/it] + + + + + +Train steps ... : 68%|████████████████████████████████████████▉ | 694/1016 [1:20:02<33:42, 6.28s/it] + + + + + +Train steps ... : 69%|█████████████████████████████████████████▎ | 699/1016 [1:20:39<40:23, 7.64s/it] + + + + + + +Train steps ... : 69%|█████████████████████████████████████████▋ | 705/1016 [1:21:18<40:23, 7.79s/it] + + + + + +Train steps ... : 70%|█████████████████████████████████████████▉ | 710/1016 [1:21:51<42:40, 8.37s/it] + + + + +Train steps ... : 70%|██████████████████████████████████████████▏ | 714/1016 [1:22:09<24:46, 4.92s/it] + + + + + + +Train steps ... : 71%|██████████████████████████████████████████▌ | 720/1016 [1:22:51<26:45, 5.42s/it] + + + + + +Train steps ... : 71%|██████████████████████████████████████████▊ | 725/1016 [1:23:28<31:03, 6.41s/it] + + + + +Train steps ... : 72%|███████████████████████████████████████████ | 729/1016 [1:23:57<38:09, 7.98s/it] + + + + + + +Train steps ... : 72%|███████████████████████████████████████████▍ | 735/1016 [1:24:33<36:11, 7.73s/it] + + + + + +Train steps ... : 73%|███████████████████████████████████████████▋ | 740/1016 [1:24:53<18:10, 3.95s/it] + + + + + +Train steps ... : 73%|███████████████████████████████████████████▉ | 745/1016 [1:25:32<23:38, 5.23s/it] + + + + + +Train steps ... : 74%|████████████████████████████████████████████▎ | 750/1016 [1:26:07<27:44, 6.26s/it] + + + + + +Train steps ... : 74%|████████████████████████████████████████████▌ | 755/1016 [1:26:39<30:47, 7.08s/it] + + + + +Train steps ... : 75%|████████████████████████████████████████████▊ | 759/1016 [1:27:00<26:26, 6.17s/it] + + + +Train steps ... : 75%|█████████████████████████████████████████████ | 762/1016 [1:27:25<30:18, 7.16s/it] +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.39s/it] +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.39s/it] + +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading weights file model.safetensors from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/model.safetensors +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "max_length": 448, + "pad_token_id": 50257, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ] +} +All model checkpoint weights were used when initializing WhisperForConditionalGeneration. +All the weights of WhisperForConditionalGeneration were initialized from the model checkpoint at distil-whisper/distil-large-v2. +If your task is similar to the task the model of the checkpoint was trained on, you can already use WhisperForConditionalGeneration for predictions without further training. +loading configuration file generation_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/generation_config.json +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "is_multilingual": true, + "lang_to_id": { + "<|af|>": 50327, + "<|am|>": 50334, + "<|ar|>": 50272, + "<|as|>": 50350, + "<|az|>": 50304, + "<|ba|>": 50355, + "<|be|>": 50330, + "<|bg|>": 50292, + "<|bn|>": 50302, + "<|bo|>": 50347, + "<|br|>": 50309, + "<|bs|>": 50315, + "<|ca|>": 50270, + "<|cs|>": 50283, + "<|cy|>": 50297, + "<|da|>": 50285, + "<|de|>": 50261, + "<|el|>": 50281, + "<|en|>": 50259, + "<|es|>": 50262, + "<|et|>": 50307, + "<|eu|>": 50310, + "<|fa|>": 50300, + "<|fi|>": 50277, + "<|fo|>": 50338, + "<|fr|>": 50265, + "<|gl|>": 50319, + "<|gu|>": 50333, + "<|haw|>": 50352, + "<|ha|>": 50354, + "<|he|>": 50279, + "<|hi|>": 50276, + "<|hr|>": 50291, + "<|ht|>": 50339, + "<|hu|>": 50286, + "<|hy|>": 50312, + "<|id|>": 50275, + "<|is|>": 50311, + "<|it|>": 50274, + "<|ja|>": 50266, + "<|jw|>": 50356, + "<|ka|>": 50329, + "<|kk|>": 50316, + "<|km|>": 50323, + "<|kn|>": 50306, + "<|ko|>": 50264, + "<|la|>": 50294, + "<|lb|>": 50345, + "<|ln|>": 50353, + "<|lo|>": 50336, + "<|lt|>": 50293, + "<|lv|>": 50301, + "<|mg|>": 50349, + "<|mi|>": 50295, + "<|mk|>": 50308, + "<|ml|>": 50296, + "<|mn|>": 50314, + "<|mr|>": 50320, + "<|ms|>": 50282, + "<|mt|>": 50343, + "<|my|>": 50346, + "<|ne|>": 50313, + "<|nl|>": 50271, + "<|nn|>": 50342, + "<|no|>": 50288, + "<|oc|>": 50328, + "<|pa|>": 50321, + "<|pl|>": 50269, + "<|ps|>": 50340, + "<|pt|>": 50267, + "<|ro|>": 50284, + "<|ru|>": 50263, + "<|sa|>": 50344, + "<|sd|>": 50332, + "<|si|>": 50322, + "<|sk|>": 50298, + "<|sl|>": 50305, + "<|sn|>": 50324, + "<|so|>": 50326, + "<|sq|>": 50317, + "<|sr|>": 50303, + "<|su|>": 50357, + "<|sv|>": 50273, + "<|sw|>": 50318, + "<|ta|>": 50287, + "<|te|>": 50299, + "<|tg|>": 50331, + "<|th|>": 50289, + "<|tk|>": 50341, + "<|tl|>": 50348, + "<|tr|>": 50268, + "<|tt|>": 50351, + "<|uk|>": 50280, + "<|ur|>": 50290, + "<|uz|>": 50337, + "<|vi|>": 50278, + "<|yi|>": 50335, + "<|yo|>": 50325, + "<|zh|>": 50260 + }, + "language": "<|en|>", + "max_initial_timestamp_index": 50, + "max_length": 448, + "no_timestamps_token_id": 50363, + "pad_token_id": 50257, + "prev_sot_token_id": 50361, + "return_timestamps": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "task": "transcribe", + "task_to_id": { + "transcribe": 50359, + "translate": 50358 + }, + "use_scan": false +} +loading file vocab.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/vocab.json +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer.json +loading file merges.txt from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/merges.txt +loading file normalizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/normalizer.json +loading file added_tokens.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/added_tokens.json +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer_config.json +Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. +loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/preprocessor_config.json +Feature extractor WhisperFeatureExtractor { + "chunk_length": 30, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 80, + "hop_length": 160, + "n_fft": 400, + "n_samples": 480000, + "nb_max_frames": 3000, + "padding_side": "right", + "padding_value": 0.0, + "processor_class": "WhisperProcessor", + "return_attention_mask": false, + "sampling_rate": 16000 +} +You have passed task=transcribe, but also have set `forced_decoder_ids` to [[1, 50259], [2, 50359], [3, 50363]] which creates a conflict. `forced_decoder_ids` will be ignored in favor of task=transcribe. +Eval results for step (762 / 1016 | Eval Loss: 4.11639928817749 | Eval clap: 0.39043399691581726 | Eval wer: 15.151515151515152 |) + + + +Train steps ... : 75%|█████████████████████████████████████████████▏ | 765/1016 [1:28:16<43:01, 10.29s/it] + + + + +Train steps ... : 76%|█████████████████████████████████████████████▍ | 769/1016 [1:28:58<45:59, 11.17s/it] + + + + + + +Train steps ... : 76%|█████████████████████████████████████████████▊ | 775/1016 [1:29:36<33:31, 8.34s/it] + + + + + +Train steps ... : 77%|██████████████████████████████████████████████ | 780/1016 [1:30:00<17:48, 4.53s/it] + + + + +Train steps ... : 77%|██████████████████████████████████████████████▎ | 784/1016 [1:30:33<25:35, 6.62s/it] + + + + + + +Train steps ... : 78%|██████████████████████████████████████████████▋ | 790/1016 [1:31:13<26:43, 7.10s/it] + + + + +Train steps ... : 78%|██████████████████████████████████████████████▉ | 794/1016 [1:31:39<30:27, 8.23s/it] + + + + + + +Train steps ... : 79%|███████████████████████████████████████████████▏ | 800/1016 [1:32:18<28:35, 7.94s/it] + + + + +Train steps ... : 79%|███████████████████████████████████████████████▍ | 804/1016 [1:32:40<19:33, 5.53s/it] + + + + + +Train steps ... : 80%|███████████████████████████████████████████████▊ | 809/1016 [1:33:18<23:29, 6.81s/it] + + + + + +Train steps ... : 80%|████████████████████████████████████████████████ | 814/1016 [1:33:52<25:45, 7.65s/it] + + + + + + +Train steps ... : 81%|████████████████████████████████████████████████▍ | 820/1016 [1:34:34<28:20, 8.67s/it] + + + + +Train steps ... : 81%|████████████████████████████████████████████████▋ | 824/1016 [1:34:48<14:17, 4.46s/it] + + + + + +Train steps ... : 82%|████████████████████████████████████████████████▉ | 829/1016 [1:35:26<16:52, 5.41s/it] + + + + + +Train steps ... : 82%|█████████████████████████████████████████████████▎ | 834/1016 [1:36:05<21:19, 7.03s/it] + + + + + + +Train steps ... : 83%|█████████████████████████████████████████████████▌ | 840/1016 [1:36:44<19:46, 6.74s/it] + + + + +Train steps ... : 83%|█████████████████████████████████████████████████▊ | 844/1016 [1:37:11<24:32, 8.56s/it] + + + + + + +Train steps ... : 84%|██████████████████████████████████████████████████▏ | 850/1016 [1:37:50<22:34, 8.16s/it] + + + + +Train steps ... : 84%|██████████████████████████████████████████████████▍ | 854/1016 [1:38:12<14:54, 5.52s/it] + + + + + + +Train steps ... : 85%|██████████████████████████████████████████████████▊ | 860/1016 [1:38:52<14:41, 5.65s/it] + + + + + +Train steps ... : 85%|███████████████████████████████████████████████████ | 865/1016 [1:39:26<16:19, 6.49s/it] + + + + +Train steps ... : 86%|███████████████████████████████████████████████████▎ | 869/1016 [1:39:53<20:38, 8.42s/it] + + + + + + +Train steps ... : 86%|███████████████████████████████████████████████████▋ | 875/1016 [1:40:32<18:33, 7.90s/it] + + + + +Train steps ... : 87%|███████████████████████████████████████████████████▉ | 879/1016 [1:40:55<12:49, 5.62s/it] + + + + + +Train steps ... : 87%|████████████████████████████████████████████████████▏ | 884/1016 [1:41:31<14:50, 6.75s/it] + + + + + +Train steps ... : 88%|████████████████████████████████████████████████████▌ | 889/1016 [1:42:07<16:49, 7.95s/it] +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.51s/it] +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.51s/it] + +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading weights file model.safetensors from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/model.safetensors +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "max_length": 448, + "pad_token_id": 50257, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ] +} +All model checkpoint weights were used when initializing WhisperForConditionalGeneration. +All the weights of WhisperForConditionalGeneration were initialized from the model checkpoint at distil-whisper/distil-large-v2. +If your task is similar to the task the model of the checkpoint was trained on, you can already use WhisperForConditionalGeneration for predictions without further training. +loading configuration file generation_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/generation_config.json +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "is_multilingual": true, + "lang_to_id": { + "<|af|>": 50327, + "<|am|>": 50334, + "<|ar|>": 50272, + "<|as|>": 50350, + "<|az|>": 50304, + "<|ba|>": 50355, + "<|be|>": 50330, + "<|bg|>": 50292, + "<|bn|>": 50302, + "<|bo|>": 50347, + "<|br|>": 50309, + "<|bs|>": 50315, + "<|ca|>": 50270, + "<|cs|>": 50283, + "<|cy|>": 50297, + "<|da|>": 50285, + "<|de|>": 50261, + "<|el|>": 50281, + "<|en|>": 50259, + "<|es|>": 50262, + "<|et|>": 50307, + "<|eu|>": 50310, + "<|fa|>": 50300, + "<|fi|>": 50277, + "<|fo|>": 50338, + "<|fr|>": 50265, + "<|gl|>": 50319, + "<|gu|>": 50333, + "<|haw|>": 50352, + "<|ha|>": 50354, + "<|he|>": 50279, + "<|hi|>": 50276, + "<|hr|>": 50291, + "<|ht|>": 50339, + "<|hu|>": 50286, + "<|hy|>": 50312, + "<|id|>": 50275, + "<|is|>": 50311, + "<|it|>": 50274, + "<|ja|>": 50266, + "<|jw|>": 50356, + "<|ka|>": 50329, + "<|kk|>": 50316, + "<|km|>": 50323, + "<|kn|>": 50306, + "<|ko|>": 50264, + "<|la|>": 50294, + "<|lb|>": 50345, + "<|ln|>": 50353, + "<|lo|>": 50336, + "<|lt|>": 50293, + "<|lv|>": 50301, + "<|mg|>": 50349, + "<|mi|>": 50295, + "<|mk|>": 50308, + "<|ml|>": 50296, + "<|mn|>": 50314, + "<|mr|>": 50320, + "<|ms|>": 50282, + "<|mt|>": 50343, + "<|my|>": 50346, + "<|ne|>": 50313, + "<|nl|>": 50271, + "<|nn|>": 50342, + "<|no|>": 50288, + "<|oc|>": 50328, + "<|pa|>": 50321, + "<|pl|>": 50269, + "<|ps|>": 50340, + "<|pt|>": 50267, + "<|ro|>": 50284, + "<|ru|>": 50263, + "<|sa|>": 50344, + "<|sd|>": 50332, + "<|si|>": 50322, + "<|sk|>": 50298, + "<|sl|>": 50305, + "<|sn|>": 50324, + "<|so|>": 50326, + "<|sq|>": 50317, + "<|sr|>": 50303, + "<|su|>": 50357, + "<|sv|>": 50273, + "<|sw|>": 50318, + "<|ta|>": 50287, + "<|te|>": 50299, + "<|tg|>": 50331, + "<|th|>": 50289, + "<|tk|>": 50341, + "<|tl|>": 50348, + "<|tr|>": 50268, + "<|tt|>": 50351, + "<|uk|>": 50280, + "<|ur|>": 50290, + "<|uz|>": 50337, + "<|vi|>": 50278, + "<|yi|>": 50335, + "<|yo|>": 50325, + "<|zh|>": 50260 + }, + "language": "<|en|>", + "max_initial_timestamp_index": 50, + "max_length": 448, + "no_timestamps_token_id": 50363, + "pad_token_id": 50257, + "prev_sot_token_id": 50361, + "return_timestamps": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "task": "transcribe", + "task_to_id": { + "transcribe": 50359, + "translate": 50358 + }, + "use_scan": false +} +loading file vocab.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/vocab.json +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer.json +loading file merges.txt from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/merges.txt +loading file normalizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/normalizer.json +loading file added_tokens.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/added_tokens.json +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer_config.json +Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. +loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/preprocessor_config.json +Feature extractor WhisperFeatureExtractor { + "chunk_length": 30, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 80, + "hop_length": 160, + "n_fft": 400, + "n_samples": 480000, + "nb_max_frames": 3000, + "padding_side": "right", + "padding_value": 0.0, + "processor_class": "WhisperProcessor", + "return_attention_mask": false, + "sampling_rate": 16000 +} +You have passed task=transcribe, but also have set `forced_decoder_ids` to [[1, 50259], [2, 50359], [3, 50363]] which creates a conflict. `forced_decoder_ids` will be ignored in favor of task=transcribe. +Eval results for step (889 / 1016 | Eval Loss: 4.112224102020264 | Eval clap: 0.3680724799633026 | Eval wer: 14.393939393939394 |) +Train steps ... : 88%|████████████████████████████████████████████████████▌ | 889/1016 [1:42:47<16:49, 7.95s/it] + + + + + +Train steps ... : 88%|████████████████████████████████████████████████████▊ | 894/1016 [1:43:18<22:13, 10.93s/it] + + + + + +Train steps ... : 88%|█████████████████████████████████████████████████████ | 899/1016 [1:44:00<17:37, 9.04s/it] + + + + + +Train steps ... : 89%|█████████████████████████████████████████████████████▍ | 904/1016 [1:44:33<15:12, 8.15s/it] + + + + + +Train steps ... : 89%|█████████████████████████████████████████████████████▋ | 909/1016 [1:45:00<12:11, 6.84s/it] + + + + + + +Train steps ... : 90%|██████████████████████████████████████████████████████ | 915/1016 [1:45:38<10:10, 6.05s/it] + + + + +Train steps ... : 90%|██████████████████████████████████████████████████████▎ | 919/1016 [1:46:07<09:51, 6.09s/it] + + + + + +Train steps ... : 91%|██████████████████████████████████████████████████████▌ | 924/1016 [1:46:43<11:22, 7.42s/it] + + + + + +Train steps ... : 91%|██████████████████████████████████████████████████████▊ | 929/1016 [1:47:15<11:20, 7.82s/it] + + + + + + +Train steps ... : 92%|███████████████████████████████████████████████████████▏ | 935/1016 [1:47:54<10:51, 8.05s/it] + + + + + +Train steps ... : 93%|███████████████████████████████████████████████████████▌ | 940/1016 [1:48:20<07:58, 6.29s/it] + + + + + +Train steps ... : 93%|███████████████████████████████████████████████████████▊ | 945/1016 [1:48:52<06:08, 5.18s/it] + + + + +Train steps ... : 93%|████████████████████████████████████████████████████████ | 949/1016 [1:49:25<08:03, 7.21s/it] + + + + + + +Train steps ... : 94%|████████████████████████████████████████████████████████▍ | 955/1016 [1:50:03<07:20, 7.22s/it] + + + + + +Train steps ... : 94%|████████████████████████████████████████████████████████▋ | 960/1016 [1:50:36<07:28, 8.02s/it] + + + + + +Train steps ... : 95%|████████████████████████████████████████████████████████▉ | 965/1016 [1:51:01<05:06, 6.02s/it] + + + + + +Train steps ... : 95%|█████████████████████████████████████████████████████████▎ | 970/1016 [1:51:32<03:43, 4.87s/it] + + + + +Train steps ... : 96%|█████████████████████████████████████████████████████████▌ | 974/1016 [1:52:05<05:03, 7.22s/it] + + + + + +Train steps ... : 96%|█████████████████████████████████████████████████████████▊ | 979/1016 [1:52:39<05:06, 8.27s/it] + + + + + +Train steps ... : 97%|██████████████████████████████████████████████████████████ | 984/1016 [1:53:08<03:56, 7.38s/it] + + + + + + +Train steps ... : 97%|██████████████████████████████████████████████████████████▍ | 990/1016 [1:53:44<02:32, 5.88s/it] + + + + + +Train steps ... : 98%|██████████████████████████████████████████████████████████▊ | 995/1016 [1:54:13<01:39, 4.74s/it] + + + + + +Train steps ... : 98%|██████████████████████████████████████████████████████████ | 1000/1016 [1:54:52<01:45, 6.59s/it] + + + + + +Train steps ... : 99%|██████████████████████████████████████████████████████████▎| 1005/1016 [1:55:28<01:22, 7.52s/it] + + + + +Train steps ... : 99%|██████████████████████████████████████████████████████████▌| 1009/1016 [1:55:51<00:50, 7.22s/it] + + + + + + +Train steps ... : 100%|██████████████████████████████████████████████████████████▉| 1015/1016 [1:56:28<00:06, 6.29s/it] + +Train steps ... : 100%|███████████████████████████████████████████████████████████| 1016/1016 [1:56:41<00:00, 8.40s/it] +Train steps ... : 100%|███████████████████████████████████████████████████████████| 1016/1016 [1:56:41<00:00, 8.40s/it]Configuration saved in ../output_dir_training_constant_concat/config.json +Configuration saved in ../output_dir_training_constant_concat/generation_config.json +Model weights saved in ../output_dir_training_constant_concat/model.safetensors +Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.31s/it] +Evaluating - Generation ...: 100%|████████████████████████████████████████████████████████| 1/1 [00:25<00:00, 25.70s/it] +It is strongly recommended to pass the `sampling_rate` argument to this function. Failing to do so can result in silent errors that might be hard to debug. +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json +Model config WhisperConfig { + "_name_or_path": "distil-whisper/distil-large-v2", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 2, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 80, + "pad_token_id": 50257, + "scale_embedding": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "torch_dtype": "float32", + "transformers_version": "4.41.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51865 +} +loading weights file model.safetensors from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/model.safetensors +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "max_length": 448, + "pad_token_id": 50257, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ] +} +All model checkpoint weights were used when initializing WhisperForConditionalGeneration. +All the weights of WhisperForConditionalGeneration were initialized from the model checkpoint at distil-whisper/distil-large-v2. +If your task is similar to the task the model of the checkpoint was trained on, you can already use WhisperForConditionalGeneration for predictions without further training. +loading configuration file generation_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/generation_config.json +Generate config GenerationConfig { + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "decoder_start_token_id": 50258, + "eos_token_id": 50257, + "forced_decoder_ids": [ + [ + 1, + 50259 + ], + [ + 2, + 50359 + ], + [ + 3, + 50363 + ] + ], + "is_multilingual": true, + "lang_to_id": { + "<|af|>": 50327, + "<|am|>": 50334, + "<|ar|>": 50272, + "<|as|>": 50350, + "<|az|>": 50304, + "<|ba|>": 50355, + "<|be|>": 50330, + "<|bg|>": 50292, + "<|bn|>": 50302, + "<|bo|>": 50347, + "<|br|>": 50309, + "<|bs|>": 50315, + "<|ca|>": 50270, + "<|cs|>": 50283, + "<|cy|>": 50297, + "<|da|>": 50285, + "<|de|>": 50261, + "<|el|>": 50281, + "<|en|>": 50259, + "<|es|>": 50262, + "<|et|>": 50307, + "<|eu|>": 50310, + "<|fa|>": 50300, + "<|fi|>": 50277, + "<|fo|>": 50338, + "<|fr|>": 50265, + "<|gl|>": 50319, + "<|gu|>": 50333, + "<|haw|>": 50352, + "<|ha|>": 50354, + "<|he|>": 50279, + "<|hi|>": 50276, + "<|hr|>": 50291, + "<|ht|>": 50339, + "<|hu|>": 50286, + "<|hy|>": 50312, + "<|id|>": 50275, + "<|is|>": 50311, + "<|it|>": 50274, + "<|ja|>": 50266, + "<|jw|>": 50356, + "<|ka|>": 50329, + "<|kk|>": 50316, + "<|km|>": 50323, + "<|kn|>": 50306, + "<|ko|>": 50264, + "<|la|>": 50294, + "<|lb|>": 50345, + "<|ln|>": 50353, + "<|lo|>": 50336, + "<|lt|>": 50293, + "<|lv|>": 50301, + "<|mg|>": 50349, + "<|mi|>": 50295, + "<|mk|>": 50308, + "<|ml|>": 50296, + "<|mn|>": 50314, + "<|mr|>": 50320, + "<|ms|>": 50282, + "<|mt|>": 50343, + "<|my|>": 50346, + "<|ne|>": 50313, + "<|nl|>": 50271, + "<|nn|>": 50342, + "<|no|>": 50288, + "<|oc|>": 50328, + "<|pa|>": 50321, + "<|pl|>": 50269, + "<|ps|>": 50340, + "<|pt|>": 50267, + "<|ro|>": 50284, + "<|ru|>": 50263, + "<|sa|>": 50344, + "<|sd|>": 50332, + "<|si|>": 50322, + "<|sk|>": 50298, + "<|sl|>": 50305, + "<|sn|>": 50324, + "<|so|>": 50326, + "<|sq|>": 50317, + "<|sr|>": 50303, + "<|su|>": 50357, + "<|sv|>": 50273, + "<|sw|>": 50318, + "<|ta|>": 50287, + "<|te|>": 50299, + "<|tg|>": 50331, + "<|th|>": 50289, + "<|tk|>": 50341, + "<|tl|>": 50348, + "<|tr|>": 50268, + "<|tt|>": 50351, + "<|uk|>": 50280, + "<|ur|>": 50290, + "<|uz|>": 50337, + "<|vi|>": 50278, + "<|yi|>": 50335, + "<|yo|>": 50325, + "<|zh|>": 50260 + }, + "language": "<|en|>", + "max_initial_timestamp_index": 50, + "max_length": 448, + "no_timestamps_token_id": 50363, + "pad_token_id": 50257, + "prev_sot_token_id": 50361, + "return_timestamps": false, + "suppress_tokens": [ + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50360, + 50361, + 50362 + ], + "task": "transcribe", + "task_to_id": { + "transcribe": 50359, + "translate": 50358 + }, + "use_scan": false +} +loading file vocab.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/vocab.json +loading file tokenizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer.json +loading file merges.txt from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/merges.txt +loading file normalizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/normalizer.json +loading file added_tokens.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/added_tokens.json +loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/special_tokens_map.json +loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer_config.json +Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. +loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/preprocessor_config.json +Feature extractor WhisperFeatureExtractor { + "chunk_length": 30, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 80, + "hop_length": 160, + "n_fft": 400, + "n_samples": 480000, + "nb_max_frames": 3000, + "padding_side": "right", + "padding_value": 0.0, + "processor_class": "WhisperProcessor", + "return_attention_mask": false, + "sampling_rate": 16000 +} +You have passed task=transcribe, but also have set `forced_decoder_ids` to [[1, 50259], [2, 50359], [3, 50363]] which creates a conflict. `forced_decoder_ids` will be ignored in favor of task=transcribe. +Train steps ... : 100%|███████████████████████████████████████████████████████████| 1016/1016 [1:57:24<00:00, 8.40s/it] +Eval results for step (1016 / 1016 | Eval Loss: 4.111780643463135 | Eval clap: 0.38305774331092834 | Eval wer: 15.909090909090908 |) \ No newline at end of file diff --git a/wandb/run-20240513_205249-qaoje1x9/files/requirements.txt b/wandb/run-20240513_205249-qaoje1x9/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..dc3da8d76890b37da99d889b5976fd39cb7ff10a --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/requirements.txt @@ -0,0 +1,225 @@ +Babel==2.15.0 +Flask-Cors==4.0.1 +Flask-RESTful==0.3.10 +Flask-SocketIO==5.3.6 +Flask==2.2.5 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +PyYAML==6.0.1 +Pygments==2.18.0 +Send2Trash==1.8.3 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.30.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +aniso8601==9.0.1 +annotated-types==0.6.0 +anyio==4.3.0 +argbind==0.3.7 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +arrow==1.3.0 +asttokens==2.4.1 +async-lru==2.0.4 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bidict==0.23.1 +bitsandbytes==0.43.1 +bleach==6.1.0 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +coloredlogs==14.0 +comm==0.2.2 +contourpy==1.2.1 +cycler==0.12.1 +datasets==2.19.1 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +descript-audio-codec==1.0.0 +descript-audiotools==0.7.2 +dill==0.3.8 +dnspython==2.3.0 +docker-pycreds==0.4.0 +docstring_parser==0.16 +editdistance==0.8.1 +einops==0.8.0 +et-xmlfile==1.1.0 +evaluate==0.4.2 +eventlet==0.36.1 +executing==2.0.1 +fastjsonschema==2.19.1 +ffmpy==0.3.2 +filelock==3.14.0 +fire==0.6.0 +flask-talisman==1.1.0 +flatten-dict==0.4.2 +fonttools==4.51.0 +fqdn==1.5.1 +frozenlist==1.4.1 +fsspec==2024.3.1 +future==1.0.0 +g2p==2.0.0 +gitdb==4.0.11 +greenlet==3.0.3 +grpcio==1.63.0 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.0 +humanfriendly==10.0 +idna==3.7 +importlib_resources==6.4.0 +ipdb==0.13.13 +ipykernel==6.29.4 +ipython==8.24.0 +isoduration==20.11.0 +itsdangerous==2.2.0 +jedi==0.19.1 +jiwer==3.0.4 +joblib==1.4.2 +json5==0.9.25 +jsonpointer==2.4 +jsonschema-specifications==2023.12.1 +jsonschema==4.22.0 +julius==0.2.7 +jupyter-events==0.10.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.1 +jupyter_core==5.7.2 +jupyter_server==2.14.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.2.0 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.1 +kiwisolver==1.4.5 +lazy_loader==0.4 +librosa==0.10.2 +llvmlite==0.42.0 +markdown-it-py==3.0.0 +markdown2==2.4.13 +matplotlib-inline==0.1.7 +matplotlib==3.8.4 +mdurl==0.1.2 +mistune==3.0.2 +mpmath==1.3.0 +msgpack==1.0.8 +multidict==6.0.5 +multiprocess==0.70.16 +munkres==1.1.4 +nbclient==0.10.0 +nbconvert==7.16.4 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.3 +notebook_shim==0.2.4 +numba==0.59.1 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.1.105 +openpyxl==3.1.2 +overrides==7.7.0 +packaging==24.0 +pandas==2.2.2 +pandocfilters==1.5.1 +panphon==0.20.0 +parler_tts==0.1 +parso==0.8.4 +pexpect==4.9.0 +pillow==10.3.0 +pip==24.0 +platformdirs==4.2.1 +pooch==1.8.1 +prometheus_client==0.20.0 +prompt-toolkit==3.0.43 +protobuf==3.19.6 +psutil==5.9.8 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==16.0.0 +pycparser==2.22 +pydantic==2.7.1 +pydantic_core==2.18.2 +pyloudnorm==0.1.1 +pyparsing==3.1.2 +pystoi==0.4.1 +python-dateutil==2.9.0.post0 +python-engineio==4.9.0 +python-json-logger==2.0.7 +python-socketio==5.11.2 +pytz==2024.1 +pyzmq==26.0.3 +randomname==0.2.1 +rapidfuzz==3.9.0 +referencing==0.35.1 +regex==2024.4.28 +requests==2.31.0 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==13.7.1 +rpds-py==0.18.1 +safetensors==0.4.3 +scikit-learn==1.4.2 +scipy==1.13.0 +sentencepiece==0.2.0 +sentry-sdk==2.1.1 +setproctitle==1.3.3 +setuptools==69.5.1 +simple-websocket==1.0.0 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.1 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +stack-data==0.6.3 +sympy==1.12 +tensorboard-data-server==0.7.2 +tensorboard==2.16.2 +termcolor==2.4.0 +terminado==0.18.1 +text-unidecode==1.3 +threadpoolctl==3.5.0 +tinycss2==1.3.0 +tokenizers==0.19.1 +torch-stoi==0.2.1 +torch==2.3.0 +torchaudio==2.3.0 +tornado==6.4 +tqdm==4.66.4 +traitlets==5.14.3 +transformers==4.41.0.dev0 +transformers==4.41.0.dev0 +triton==2.3.0 +types-python-dateutil==2.9.0.20240316 +typing_extensions==4.11.0 +tzdata==2024.1 +unicodecsv==0.14.1 +uri-template==1.3.0 +urllib3==2.2.1 +wandb==0.17.0 +wcwidth==0.2.13 +webcolors==1.13 +webencodings==0.5.1 +websocket-client==1.8.0 +wheel==0.43.0 +wsproto==1.2.0 +xxhash==3.4.1 +yarl==1.9.4 \ No newline at end of file diff --git a/wandb/run-20240513_205249-qaoje1x9/files/wandb-metadata.json b/wandb/run-20240513_205249-qaoje1x9/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..88613d3a609246f0f3d056620ae34005f0e05a47 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/wandb-metadata.json @@ -0,0 +1,706 @@ +{ + "os": "Linux-5.4.0-166-generic-x86_64-with-glibc2.31", + "python": "3.11.9", + "heartbeatAt": "2024-05-13T18:52:49.548604", + "startedAt": "2024-05-13T18:52:49.014517", + "docker": null, + "cuda": null, + "args": [ + "finetuning_concatenated_config.json" + ], + "state": "running", + "program": "/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py", + "codePathLocal": "run_parler_tts_training.py", + "codePath": "run_parler_tts_training.py", + "git": { + "remote": "https://huggingface.co/sanchit-gandhi/parler-tts-mini-v0.1-expresso-concatenated-combined", + "commit": "50ba4323d7b8bb052629aa1b88283b9df081a821" + }, + "email": "sanchit@huggingface.co", + "root": "/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined", + "host": "hf-dgx-01", + "username": "sanchit", + "executable": "/home/sanchit/miniconda3/envs/venv/bin/python", + "cpu_count": 64, + "cpu_count_logical": 128, + "cpu_freq": { + "current": 2360.7500156249994, + "min": 1500.0, + "max": 2250.0 + }, + "cpu_freq_per_core": [ + { + "current": 2426.568, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2649.263, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3020.85, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1796.979, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3294.864, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2097.135, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1662.773, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1664.784, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3332.909, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1661.756, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1583.69, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3225.763, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1713.916, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2383.108, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1713.839, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1712.972, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2360.612, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3295.124, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3286.553, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3279.781, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1793.151, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1796.308, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1792.785, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1791.778, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1661.19, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3194.481, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3332.944, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3210.408, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2661.36, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1716.914, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1712.659, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1717.873, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1797.636, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1796.013, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2777.317, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1792.753, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3297.674, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2290.065, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1670.248, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1665.016, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3299.101, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1647.783, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1660.513, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1659.949, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3318.509, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2655.27, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3320.021, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3295.518, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3341.592, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1662.725, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3295.592, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1662.043, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1714.097, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3186.226, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1712.114, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1718.52, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1701.556, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3294.711, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3324.584, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1659.515, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1794.143, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1796.233, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1795.995, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1793.883, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1790.024, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1794.313, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2988.012, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1792.457, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3328.716, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1661.634, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1662.011, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2209.287, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3295.369, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1658.97, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1662.145, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2424.213, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2703.578, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2305.348, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1716.951, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1718.102, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3140.126, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3317.716, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3313.497, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3316.184, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1809.855, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3078.611, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3080.162, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2037.58, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2198.863, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3263.675, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3293.238, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2189.229, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2038.013, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2217.719, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3135.886, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2193.004, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1805.953, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1832.919, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1831.97, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1831.656, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3312.099, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2035.327, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1957.443, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2216.063, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3022.623, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1684.605, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1684.549, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1702.372, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3286.386, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3288.738, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3278.751, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3290.555, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3292.535, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2153.843, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3293.152, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1644.116, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1655.185, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3070.901, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1655.504, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1655.465, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1719.19, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3284.274, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 3283.49, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1645.649, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1790.709, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 2223.188, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1797.667, + "min": 1500.0, + "max": 2250.0 + }, + { + "current": 1796.254, + "min": 1500.0, + "max": 2250.0 + } + ], + "disk": { + "/": { + "total": 1757.8785285949707, + "used": 1663.5053825378418 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 5, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA DGX Display", + "memory_total": 4294967296 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 503.5396919250488 + } +} diff --git a/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json b/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..42f19de9b96830bbae23d807d1645a421d8ba786 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/loss": 4.33385705947876, "train/time": 6743.537756443024, "train/epoch": 7.9921259842519685, "train/learning_rate": 3.3641208838641747e-10, "_timestamp": 1715634092.0433934, "_runtime": 7723.0233154296875, "_step": 1016, "predictions/eval-step-127": {"_type": "table-file", "sha256": "a3f59299480689ee898f135ed6e81fe730f366e27845d1e68cb51b42bafbb600", "size": 2497, "artifact_path": "wandb-client-artifact://r65s7mfzzpzamfxbc89ngja1l9rcfm59uy7n8qrrxma4nrejykpujze5p92zgbtr71zaejsgvlcpza9oudtyvu0jj4kxz9kyhsx5zv3vfzbtwk3mziil29nbqeo097vy/predictions/eval-step-127.table.json", "_latest_artifact_path": "wandb-client-artifact://xj3jam3fyktc32etn5p9gkhwwfkokoax5oii7ilm491vmr4ogffak146q3n8bl58ak4r82kajrfod04yzf55jd8kt1c2mbkvz67s6qtzj75g8r8ugtig4w6z7dwzkqjc:latest/predictions/eval-step-127.table.json", "path": "media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json", "ncols": 3, "nrows": 7}, "Speech samples/eval": {"_type": "audio", "count": 7, "audio": [{"_type": "audio-file", "sha256": "9da1fed11be9d614d9ecceb9a3d8bd45a3539c36149b2301d2bc9b4e032499aa", "size": 961580, "path": "media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav", "caption": "\"Well,\" said he, when Franz had concluded, \"what do you find to object to in all you have related? --- DESCRIPTION: A female speaker with a moderate pitch and slightly monotone tone delivers her words at a moderate speed in a confined environment with only slight background noise present."}, {"_type": "audio-file", "sha256": "3ed6544e58dd861a5d9e8297001b2e4939e176f1dfa6cf48604f1fd2e2b0bdb6", "size": 961580, "path": "media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav", "caption": "The stop at Queenstown, the tedious passage up the Mersey, were things that he noted dimly through his growing impatience. --- DESCRIPTION: A female speaker with a slightly low-pitched, quite monotone voice delivers her words at a slightly faster than average pace in a confined environment with minimal background noise."}, {"_type": "audio-file", "sha256": "3c9adbd9374e0fb5ce3d60fef881f3c5afdc9c749897ad144e0fdd8b4485031c", "size": 961580, "path": "media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav", "caption": "'Yet I thought,' she said, 'that no man could have done this deed but Sigurd, Fafnir's bane, who was my true love. --- DESCRIPTION: A female speaker with a slightly low-pitched voice delivers her words quite rapidly and with only slight monotone intonation in a confined, almost echo-free environment."}, {"_type": "audio-file", "sha256": "345bfb6a72849809d361d89b486b992a530bb6b14e5e82f395fbd8fe5e7f4277", "size": 961580, "path": "media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav", "caption": "Built with a lightning bolt? You got it! Sorry I couldn't provide more info! And Roland said, oh, he's so funny. --- DESCRIPTION: Thomas speaks with clear and deliberate enunciation at a quite slow pace with high quality."}, {"_type": "audio-file", "sha256": "3cd94e4824cc6c8fb09c879ae537cea055e30a34f0cc5fc6414cd871d4ba64df", "size": 961580, "path": "media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav", "caption": "What's the latest from the All Pakistan Drama Page? How'd your shirt get torn, sweetie? --- DESCRIPTION: Jerry speaks slightly slowly and with confusion in his voice with high quality."}, {"_type": "audio-file", "sha256": "193b029d494fd24e7cfa1fec8064d5f9d5a8471b50e074ad6a52018a3d49ec7b", "size": 961580, "path": "media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav", "caption": "I started to get involved as a Party Leader nearly a quarter of a century ago. --- DESCRIPTION: Jenny's voice is quite monotone and her speech is delivered very fast in a confined space with only slight ambient sound."}, {"_type": "audio-file", "sha256": "ec838b0233dbe87d33f3d525c38fd0a25be427592d85b665bc656a472a265c53", "size": 961580, "path": "media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav", "caption": "He started up nervously from the stone block for he could no longer quench the flame in his blood. --- DESCRIPTION: A male voice with a low pitch and monotone tone speaks quite fast in a confined and clear recording environment."}], "sampleRates": [44100, 44100, 44100, 44100, 44100, 44100, 44100], "durations": [10.901768707482994, 10.901768707482994, 10.901768707482994, 10.901768707482994, 10.901768707482994, 10.901768707482994, 10.901768707482994], "captions": ["\"Well,\" said he, when Franz had concluded, \"what do you find to object to in all you have related? --- DESCRIPTION: A female speaker with a moderate pitch and slightly monotone tone delivers her words at a moderate speed in a confined environment with only slight background noise present.", "The stop at Queenstown, the tedious passage up the Mersey, were things that he noted dimly through his growing impatience. --- DESCRIPTION: A female speaker with a slightly low-pitched, quite monotone voice delivers her words at a slightly faster than average pace in a confined environment with minimal background noise.", "'Yet I thought,' she said, 'that no man could have done this deed but Sigurd, Fafnir's bane, who was my true love. --- DESCRIPTION: A female speaker with a slightly low-pitched voice delivers her words quite rapidly and with only slight monotone intonation in a confined, almost echo-free environment.", "Built with a lightning bolt? You got it! Sorry I couldn't provide more info! And Roland said, oh, he's so funny. --- DESCRIPTION: Thomas speaks with clear and deliberate enunciation at a quite slow pace with high quality.", "What's the latest from the All Pakistan Drama Page? How'd your shirt get torn, sweetie? --- DESCRIPTION: Jerry speaks slightly slowly and with confusion in his voice with high quality.", "I started to get involved as a Party Leader nearly a quarter of a century ago. --- DESCRIPTION: Jenny's voice is quite monotone and her speech is delivered very fast in a confined space with only slight ambient sound.", "He started up nervously from the stone block for he could no longer quench the flame in his blood. --- DESCRIPTION: A male voice with a low pitch and monotone tone speaks quite fast in a confined and clear recording environment."]}, "eval/loss": 4.111780643463135, "eval/clap": 0.38305774331092834, "eval/wer": 15.909090909090908, "eval/time": 28.344980716705322, "eval/epoch": 8.0, "predictions/eval-step-254": {"_type": "table-file", "sha256": "eb7b50d0179e2d66397e1d8b2a8ee300146fc205e7986e792698adb5b14b3da4", "size": 2490, "artifact_path": "wandb-client-artifact://gxezbynsognys4dwtc0bxc6t10x81ofprs8sbci803i45p5v7gnfjxic4lk1vu4lg93zpu0scsveyznoyxwf5o967vtfl1xavvdhulhgdtrls11mimpk7hiyl0yxy2o9/predictions/eval-step-254.table.json", "_latest_artifact_path": "wandb-client-artifact://hq1w33x8g7b99brqjf61041n6sx12itdqfbggvl5qvdpyvsctnd52v6wsl1cbk0olbcmtxnnkerd82gsulcm4sdtl7yf17q79bgmvrlxf0vmocdjmlpyklegt2w6zgsk:latest/predictions/eval-step-254.table.json", "path": "media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json", "ncols": 3, "nrows": 7}, "predictions/eval-step-381": {"_type": "table-file", "sha256": "1bf220a613a57386890ad7d28a38e7977de741f1610ca1628197e3dc926002a4", "size": 2538, "artifact_path": "wandb-client-artifact://psonrbao2jklltxzg6h1cogx519pup124w5lxcm0hjytbmof7115u1kncbjjyt6z5juyfrg31drqmnakrhhk4rz81dlictv65zgviy1omjn3kwld205acvyz942555v8/predictions/eval-step-381.table.json", "_latest_artifact_path": "wandb-client-artifact://05ivb6mvut2vpbg0mcru8oqegxb82119or9npxkrnfs3x1ioda8xohs4t4g7asdslwzixrwuldetumq24njlu0gyesfcwa8agnstgfkb8ajlljeql16asd4e9kt9izpz:latest/predictions/eval-step-381.table.json", "path": "media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json", "ncols": 3, "nrows": 7}, "predictions/eval-step-508": {"_type": "table-file", "sha256": "ebd8a6e2299806a596d7e2ee74e815ccd9eda09b7016b7feb6841290b1eadaec", "size": 2516, "artifact_path": "wandb-client-artifact://m6yzqe8evf1xdqbchjqwb99weok3pzwgap2xvvsklicole29japz8evvok3jvtw7xfukguea2e3huxx625clk7t7pmqovp3fax9uh6go3fpdiv25lqrd46y5hlr21dcs/predictions/eval-step-508.table.json", "_latest_artifact_path": "wandb-client-artifact://yaqnve1xvyap1syncq9qfilh5p8gz45l8x1hsb7utvgjqlowjajuxyiuxu8bdti4z2jbcvwi2i9sqmqhw9aefu9xktroakdr90ja2kjyx1yjoyk7q4oos9l1t2cfrgan:latest/predictions/eval-step-508.table.json", "path": "media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json", "ncols": 3, "nrows": 7}, "predictions/eval-step-635": {"_type": "table-file", "sha256": "8b369ee796f15026de8f86bacb61934f47159b203fe59411f23a187089597c8e", "size": 2527, "artifact_path": "wandb-client-artifact://cztuhj1k3fvinf32frur17faxpc4rkvi37rrphykaurss87p1jxc6sq5ohgn92jbsx9ca4twq7gj7pjhtlu87y4jj7ru75728tvr3btkakwiq1ox9f5eopwdkqk8m449/predictions/eval-step-635.table.json", "_latest_artifact_path": "wandb-client-artifact://1q4chz0dqsbejwfycfnt9xyhrpxpkexv74h4fh7gamp9jolk6aqn8lilgo55js0cc9ld19audyxdcjtgf1yi3pz7306ckbx5d7wn23tfr0u6v4nyj5atewexdk1t7vx3:latest/predictions/eval-step-635.table.json", "path": "media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json", "ncols": 3, "nrows": 7}, "predictions/eval-step-762": {"_type": "table-file", "sha256": "df928dba2c214ae9137f4a06222760f4e79d2a3942b20dfd52406b55033ba3ec", "size": 2536, "artifact_path": "wandb-client-artifact://5y8jh20vo5dj65p88sxcs43frh807em1upuye5plrspnfpr9eyomf8u76do6xa9zyo51df83ezjzqrz8oywycvaou4aj9utd9y9a0o4nwqilch1w6t7e11wulknximmx/predictions/eval-step-762.table.json", "_latest_artifact_path": "wandb-client-artifact://eldqlw8777h5tfvttggo6lbn3k4jhg5yxm1z3i0xwad4xpngy86g9ug8dynilck7j3hbo9xfpinyidny61dmhrx1ji1uen2jjakjxn2zt8w6xv0uk0qgu3kqun5rtnke:latest/predictions/eval-step-762.table.json", "path": "media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json", "ncols": 3, "nrows": 7}, "predictions/eval-step-889": {"_type": "table-file", "sha256": "9d5bd9e7754ccad0a997568660d6f5ef77cf4c16b5a3ead09b6af63e929e3afc", "size": 2539, "artifact_path": "wandb-client-artifact://cmk95s49r1rrfa3gqoycipf9e44uuyibxpniv3s9k9rdknh68fjrrltcyum4yiz9gzeuadslyqz1bpy45rhbgz1dbhddvvee1ybwca38q1464uson9hmer44l33cz7as/predictions/eval-step-889.table.json", "_latest_artifact_path": "wandb-client-artifact://bwo96o3iefbnd2wbixyz7r0o6qaq89c7dt79czlkcvmmr0e1gfe7t04n56upo70xichx2kths5w1msbjpaa4hg3f3enqlhdz5cs47p6g6hzz0ru4spem6cri3r8w87z6:latest/predictions/eval-step-889.table.json", "path": "media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json", "ncols": 3, "nrows": 7}, "_wandb": {"runtime": 7723}, "predictions/eval-step-1k": {"_type": "table-file", "sha256": "6ea072f5a546055d87953ae6a4b784f4eb96e21885e5b5743c32a9840260c5a9", "size": 2535, "artifact_path": "wandb-client-artifact://qpmk599pj2j3eq153tnvgj54z31ffqw9bwjbvuqhj5k52a7c0tjvd3b0j091kcdksmmlebem2trdvgbnhn4h60iqfszizh1odjelmpk8mk5dp04grxk6o6bgklhjdltb/predictions/eval-step-1k.table.json", "_latest_artifact_path": "wandb-client-artifact://rtzpmyldfezh5d1p1wd4wn0e1hkr58mcutfv0fz07i2vxaxtcmgrhpgbnflhh3syp6gekj68tp694qix1g1v0ymqez3i44845ohugvsm7h83zas27gu8xp5qmvqozvgz:latest/predictions/eval-step-1k.table.json", "path": "media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json", "ncols": 3, "nrows": 7}} \ No newline at end of file diff --git a/wandb/run-20240513_205249-qaoje1x9/logs/debug-internal.log b/wandb/run-20240513_205249-qaoje1x9/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..10fc533ac41b82f6c553d78e06ac3eca75321b6b --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/logs/debug-internal.log @@ -0,0 +1,11809 @@ +2024-05-13 20:52:49,022 INFO StreamThr :1257956 [internal.py:wandb_internal():85] W&B internal server running at pid: 1257956, started at: 2024-05-13 20:52:49.021573 +2024-05-13 20:52:49,024 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status +2024-05-13 20:52:49,026 INFO WriterThread:1257956 [datastore.py:open_for_write():87] open: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/run-qaoje1x9.wandb +2024-05-13 20:52:49,027 DEBUG SenderThread:1257956 [sender.py:send():378] send: header +2024-05-13 20:52:49,027 DEBUG SenderThread:1257956 [sender.py:send():378] send: run +2024-05-13 20:52:49,408 INFO SenderThread:1257956 [dir_watcher.py:__init__():211] watching files in: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files +2024-05-13 20:52:49,408 INFO SenderThread:1257956 [sender.py:_start_run_threads():1123] run started: qaoje1x9 with start time 1715626369.020078 +2024-05-13 20:52:49,414 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: check_version +2024-05-13 20:52:49,414 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: check_version +2024-05-13 20:52:49,480 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: run_start +2024-05-13 20:52:49,502 DEBUG HandlerThread:1257956 [system_info.py:__init__():26] System info init +2024-05-13 20:52:49,502 DEBUG HandlerThread:1257956 [system_info.py:__init__():41] System info init done +2024-05-13 20:52:49,502 INFO HandlerThread:1257956 [system_monitor.py:start():194] Starting system monitor +2024-05-13 20:52:49,502 INFO SystemMonitor:1257956 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-13 20:52:49,502 INFO HandlerThread:1257956 [system_monitor.py:probe():214] Collecting system info +2024-05-13 20:52:49,503 INFO SystemMonitor:1257956 [interfaces.py:start():188] Started cpu monitoring +2024-05-13 20:52:49,503 INFO SystemMonitor:1257956 [interfaces.py:start():188] Started disk monitoring +2024-05-13 20:52:49,504 INFO SystemMonitor:1257956 [interfaces.py:start():188] Started gpu monitoring +2024-05-13 20:52:49,505 INFO SystemMonitor:1257956 [interfaces.py:start():188] Started memory monitoring +2024-05-13 20:52:49,506 INFO SystemMonitor:1257956 [interfaces.py:start():188] Started network monitoring +2024-05-13 20:52:49,548 DEBUG HandlerThread:1257956 [system_info.py:probe():150] Probing system +2024-05-13 20:52:49,550 DEBUG HandlerThread:1257956 [system_info.py:_probe_git():135] Probing git +2024-05-13 20:52:49,555 DEBUG HandlerThread:1257956 [system_info.py:_probe_git():143] Probing git done +2024-05-13 20:52:49,555 DEBUG HandlerThread:1257956 [system_info.py:probe():198] Probing system done +2024-05-13 20:52:49,555 DEBUG HandlerThread:1257956 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.31', 'python': '3.11.9', 'heartbeatAt': '2024-05-13T18:52:49.548604', 'startedAt': '2024-05-13T18:52:49.014517', 'docker': None, 'cuda': None, 'args': ('finetuning_concatenated_config.json',), 'state': 'running', 'program': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py', 'codePathLocal': 'run_parler_tts_training.py', 'codePath': 'run_parler_tts_training.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/parler-tts-mini-v0.1-expresso-concatenated-combined', 'commit': '50ba4323d7b8bb052629aa1b88283b9df081a821'}, 'email': 'sanchit@huggingface.co', 'root': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/miniconda3/envs/venv/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2360.7500156249994, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 2426.568, 'min': 1500.0, 'max': 2250.0}, {'current': 2649.263, 'min': 1500.0, 'max': 2250.0}, {'current': 3020.85, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.979, 'min': 1500.0, 'max': 2250.0}, {'current': 3294.864, 'min': 1500.0, 'max': 2250.0}, {'current': 2097.135, 'min': 1500.0, 'max': 2250.0}, {'current': 1662.773, 'min': 1500.0, 'max': 2250.0}, {'current': 1664.784, 'min': 1500.0, 'max': 2250.0}, {'current': 3332.909, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.756, 'min': 1500.0, 'max': 2250.0}, {'current': 1583.69, 'min': 1500.0, 'max': 2250.0}, {'current': 3225.763, 'min': 1500.0, 'max': 2250.0}, {'current': 1713.916, 'min': 1500.0, 'max': 2250.0}, {'current': 2383.108, 'min': 1500.0, 'max': 2250.0}, {'current': 1713.839, 'min': 1500.0, 'max': 2250.0}, {'current': 1712.972, 'min': 1500.0, 'max': 2250.0}, {'current': 2360.612, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.124, 'min': 1500.0, 'max': 2250.0}, {'current': 3286.553, 'min': 1500.0, 'max': 2250.0}, {'current': 3279.781, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.151, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.308, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.785, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.778, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.19, 'min': 1500.0, 'max': 2250.0}, {'current': 3194.481, 'min': 1500.0, 'max': 2250.0}, {'current': 3332.944, 'min': 1500.0, 'max': 2250.0}, {'current': 3210.408, 'min': 1500.0, 'max': 2250.0}, {'current': 2661.36, 'min': 1500.0, 'max': 2250.0}, {'current': 1716.914, 'min': 1500.0, 'max': 2250.0}, {'current': 1712.659, 'min': 1500.0, 'max': 2250.0}, {'current': 1717.873, 'min': 1500.0, 'max': 2250.0}, {'current': 1797.636, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.013, 'min': 1500.0, 'max': 2250.0}, {'current': 2777.317, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.753, 'min': 1500.0, 'max': 2250.0}, {'current': 3297.674, 'min': 1500.0, 'max': 2250.0}, {'current': 2290.065, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.248, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.016, 'min': 1500.0, 'max': 2250.0}, {'current': 3299.101, 'min': 1500.0, 'max': 2250.0}, {'current': 1647.783, 'min': 1500.0, 'max': 2250.0}, {'current': 1660.513, 'min': 1500.0, 'max': 2250.0}, {'current': 1659.949, 'min': 1500.0, 'max': 2250.0}, {'current': 3318.509, 'min': 1500.0, 'max': 2250.0}, {'current': 2655.27, 'min': 1500.0, 'max': 2250.0}, {'current': 3320.021, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.518, 'min': 1500.0, 'max': 2250.0}, {'current': 3341.592, 'min': 1500.0, 'max': 2250.0}, {'current': 1662.725, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.592, 'min': 1500.0, 'max': 2250.0}, {'current': 1662.043, 'min': 1500.0, 'max': 2250.0}, {'current': 1714.097, 'min': 1500.0, 'max': 2250.0}, {'current': 3186.226, 'min': 1500.0, 'max': 2250.0}, {'current': 1712.114, 'min': 1500.0, 'max': 2250.0}, {'current': 1718.52, 'min': 1500.0, 'max': 2250.0}, {'current': 1701.556, 'min': 1500.0, 'max': 2250.0}, {'current': 3294.711, 'min': 1500.0, 'max': 2250.0}, {'current': 3324.584, 'min': 1500.0, 'max': 2250.0}, {'current': 1659.515, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.143, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.233, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.995, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.883, 'min': 1500.0, 'max': 2250.0}, {'current': 1790.024, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.313, 'min': 1500.0, 'max': 2250.0}, {'current': 2988.012, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.457, 'min': 1500.0, 'max': 2250.0}, {'current': 3328.716, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.634, 'min': 1500.0, 'max': 2250.0}, {'current': 1662.011, 'min': 1500.0, 'max': 2250.0}, {'current': 2209.287, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.369, 'min': 1500.0, 'max': 2250.0}, {'current': 1658.97, 'min': 1500.0, 'max': 2250.0}, {'current': 1662.145, 'min': 1500.0, 'max': 2250.0}, {'current': 2424.213, 'min': 1500.0, 'max': 2250.0}, {'current': 2703.578, 'min': 1500.0, 'max': 2250.0}, {'current': 2305.348, 'min': 1500.0, 'max': 2250.0}, {'current': 1716.951, 'min': 1500.0, 'max': 2250.0}, {'current': 1718.102, 'min': 1500.0, 'max': 2250.0}, {'current': 3140.126, 'min': 1500.0, 'max': 2250.0}, {'current': 3317.716, 'min': 1500.0, 'max': 2250.0}, {'current': 3313.497, 'min': 1500.0, 'max': 2250.0}, {'current': 3316.184, 'min': 1500.0, 'max': 2250.0}, {'current': 1809.855, 'min': 1500.0, 'max': 2250.0}, {'current': 3078.611, 'min': 1500.0, 'max': 2250.0}, {'current': 3080.162, 'min': 1500.0, 'max': 2250.0}, {'current': 2037.58, 'min': 1500.0, 'max': 2250.0}, {'current': 2198.863, 'min': 1500.0, 'max': 2250.0}, {'current': 3263.675, 'min': 1500.0, 'max': 2250.0}, {'current': 3293.238, 'min': 1500.0, 'max': 2250.0}, {'current': 2189.229, 'min': 1500.0, 'max': 2250.0}, {'current': 2038.013, 'min': 1500.0, 'max': 2250.0}, {'current': 2217.719, 'min': 1500.0, 'max': 2250.0}, {'current': 3135.886, 'min': 1500.0, 'max': 2250.0}, {'current': 2193.004, 'min': 1500.0, 'max': 2250.0}, {'current': 1805.953, 'min': 1500.0, 'max': 2250.0}, {'current': 1832.919, 'min': 1500.0, 'max': 2250.0}, {'current': 1831.97, 'min': 1500.0, 'max': 2250.0}, {'current': 1831.656, 'min': 1500.0, 'max': 2250.0}, {'current': 3312.099, 'min': 1500.0, 'max': 2250.0}, {'current': 2035.327, 'min': 1500.0, 'max': 2250.0}, {'current': 1957.443, 'min': 1500.0, 'max': 2250.0}, {'current': 2216.063, 'min': 1500.0, 'max': 2250.0}, {'current': 3022.623, 'min': 1500.0, 'max': 2250.0}, {'current': 1684.605, 'min': 1500.0, 'max': 2250.0}, {'current': 1684.549, 'min': 1500.0, 'max': 2250.0}, {'current': 1702.372, 'min': 1500.0, 'max': 2250.0}, {'current': 3286.386, 'min': 1500.0, 'max': 2250.0}, {'current': 3288.738, 'min': 1500.0, 'max': 2250.0}, {'current': 3278.751, 'min': 1500.0, 'max': 2250.0}, {'current': 3290.555, 'min': 1500.0, 'max': 2250.0}, {'current': 3292.535, 'min': 1500.0, 'max': 2250.0}, {'current': 2153.843, 'min': 1500.0, 'max': 2250.0}, {'current': 3293.152, 'min': 1500.0, 'max': 2250.0}, {'current': 1644.116, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.185, 'min': 1500.0, 'max': 2250.0}, {'current': 3070.901, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.504, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.465, 'min': 1500.0, 'max': 2250.0}, {'current': 1719.19, 'min': 1500.0, 'max': 2250.0}, {'current': 3284.274, 'min': 1500.0, 'max': 2250.0}, {'current': 3283.49, 'min': 1500.0, 'max': 2250.0}, {'current': 1645.649, 'min': 1500.0, 'max': 2250.0}, {'current': 1790.709, 'min': 1500.0, 'max': 2250.0}, {'current': 2223.188, 'min': 1500.0, 'max': 2250.0}, {'current': 1797.667, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.254, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1663.5053825378418}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} +2024-05-13 20:52:49,556 INFO HandlerThread:1257956 [system_monitor.py:probe():224] Finished collecting system info +2024-05-13 20:52:49,556 INFO HandlerThread:1257956 [system_monitor.py:probe():227] Publishing system info +2024-05-13 20:52:49,556 DEBUG HandlerThread:1257956 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-13 20:52:49,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:49,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:50,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/conda-environment.yaml +2024-05-13 20:52:51,976 DEBUG HandlerThread:1257956 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-13 20:52:51,977 INFO HandlerThread:1257956 [system_monitor.py:probe():229] Finished publishing system info +2024-05-13 20:52:51,985 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 20:52:51,985 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-metadata.json with policy now +2024-05-13 20:52:52,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: python_packages +2024-05-13 20:52:52,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:52:52,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: python_packages +2024-05-13 20:52:52,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:52:52,322 DEBUG SenderThread:1257956 [sender.py:send():378] send: telemetry +2024-05-13 20:52:52,323 DEBUG SenderThread:1257956 [sender.py:send():378] send: config +2024-05-13 20:52:52,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/conda-environment.yaml +2024-05-13 20:52:52,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/requirements.txt +2024-05-13 20:52:52,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:52:52,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-metadata.json +2024-05-13 20:52:52,549 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ht5mod1s-wandb-metadata.json +2024-05-13 20:52:52,583 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:52,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:54,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:52:54,409 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:52:54,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:54,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:56,410 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:52:56,662 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:56,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:58,410 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:52:59,391 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:52:59,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:00,001 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:00,411 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:01,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:01,488 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:02,411 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:04,517 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:04,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:05,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:06,413 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:06,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:06,606 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:07,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:53:07,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:53:08,413 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:09,439 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:09,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:10,414 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:11,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:11,492 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:11,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:12,414 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:14,329 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:14,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:16,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:16,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:16,416 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:17,037 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:18,417 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:19,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:19,437 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:20,417 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:21,488 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:21,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:22,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:53:22,103 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:53:22,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:22,418 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:24,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:24,418 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:24,418 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/config.yaml +2024-05-13 20:53:24,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:26,419 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:28,231 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:53:28,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:28,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:28,419 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:29,250 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:30,420 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:31,218 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:31,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:32,420 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:33,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:33,316 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:35,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:36,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:36,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:36,422 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:37,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:53:37,103 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:53:38,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:38,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:38,422 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:40,277 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:41,397 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:41,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:42,423 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:43,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:43,460 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:44,424 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:46,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:46,761 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:46,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:48,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:48,925 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:49,506 DEBUG SystemMonitor:1257956 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-13 20:53:49,510 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:53:50,425 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:50,968 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:50,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:51,632 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:52,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:53:52,103 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:53:52,426 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:53,050 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:53,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:56,208 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:56,234 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:56,427 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:53:56,713 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:53:58,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:58,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:53:58,428 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:00,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:00,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:00,428 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:02,417 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:02,457 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:02,685 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:06,430 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:06,506 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:54:06,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:07,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:54:07,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:54:07,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:08,298 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:09,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:09,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:10,431 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:11,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:12,004 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:12,432 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:13,583 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:14,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:14,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:14,433 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:16,085 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:16,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:16,433 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:18,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:18,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:18,434 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:18,599 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:19,512 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:54:20,171 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:20,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:20,435 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:22,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:54:22,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:54:22,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:22,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:22,435 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:23,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:24,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:24,298 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:24,436 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:26,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:26,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:26,436 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:28,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:28,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:28,437 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:28,748 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:30,437 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:30,437 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:30,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:32,438 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:32,493 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:32,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:33,845 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:34,439 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:34,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:34,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:36,439 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:36,864 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:36,884 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:37,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:54:37,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:54:38,440 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:38,906 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:40,368 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:54:40,440 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:41,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:41,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:42,441 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:43,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:43,506 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:44,057 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:44,442 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:45,533 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:45,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:46,442 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:47,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:47,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:48,443 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:49,393 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:49,514 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:54:49,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:49,636 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:50,443 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:51,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:51,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:52,103 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:54:52,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:54:52,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:53,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:53,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:54,445 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:54,447 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:54:56,107 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:56,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:56,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:58,373 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:58,408 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:54:58,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:54:59,538 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:00,447 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:00,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:00,608 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:02,449 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:02,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:02,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:04,451 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:04,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:04,959 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:05,004 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:06,454 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:07,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:55:07,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:55:07,138 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:07,162 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:08,455 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:09,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:09,337 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:09,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:10,457 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:11,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:11,487 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:12,458 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:14,460 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:14,935 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:15,713 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:55:15,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:15,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:16,463 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:17,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:17,863 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:18,463 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:19,518 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:55:19,942 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:19,965 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:20,044 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:20,465 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:22,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:55:22,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:55:22,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:22,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:22,467 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:24,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:24,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:24,469 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:25,093 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:26,305 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:26,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:26,472 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:28,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:28,429 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:28,473 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:30,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:30,475 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:30,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:30,643 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:32,476 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:32,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:32,850 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:34,479 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:35,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:35,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:35,355 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:36,481 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:37,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:55:37,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:55:37,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:37,224 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:38,482 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:39,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:39,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:40,443 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:40,484 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:41,485 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:41,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:42,486 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:43,542 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:43,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:44,487 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:45,544 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:45,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:45,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:46,490 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:48,492 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:49,520 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:55:49,986 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:55:50,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:50,050 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:50,494 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:50,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:52,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:55:52,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:55:52,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:52,242 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:52,495 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:54,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:54,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:54,497 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:55,845 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:55:56,499 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:56,574 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:56,608 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:58,500 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:55:58,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:55:58,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:00,501 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:00,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:00,720 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:00,925 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:02,502 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:02,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:02,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:04,503 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:04,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:04,809 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:06,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:06,506 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:06,832 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:06,845 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:07,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:56:07,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:56:08,507 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:08,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:08,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:10,509 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:10,917 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:10,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:12,056 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:12,511 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:14,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:14,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:14,512 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:16,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:16,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:16,515 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:17,062 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:18,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:18,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:18,516 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:19,522 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:56:20,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:20,172 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:20,517 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:22,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:56:22,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:56:22,337 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:22,518 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:24,520 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:25,183 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:56:25,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:25,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:26,523 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:27,278 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:27,291 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:27,543 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:28,524 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:30,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:30,437 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:30,525 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:32,456 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:32,466 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:32,527 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:32,647 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:34,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:34,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:34,529 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:36,532 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:36,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:36,563 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:37,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:56:37,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:56:37,842 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:38,533 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:39,488 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:39,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:40,534 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:41,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:41,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:42,536 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:42,892 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:44,538 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:44,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:44,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:46,540 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:46,659 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:46,671 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:47,973 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:48,542 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:49,524 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:56:49,958 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:49,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:50,543 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:52,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:52,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:52,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:56:52,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:56:52,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:53,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:54,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:54,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:54,546 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:56,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:56,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:56,550 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:56:58,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:58,178 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:56:58,224 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:56:58,551 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:00,553 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:01,977 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:57:02,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:02,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:02,554 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:03,276 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:04,556 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:05,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:05,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:06,559 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:07,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:07,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:07,104 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:07,121 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:08,424 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:08,560 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:09,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:09,152 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:10,562 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:11,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:11,198 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:12,564 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:13,498 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:14,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:14,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:14,565 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:16,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:16,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:16,568 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:18,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:18,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:18,570 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:18,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:19,526 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:57:20,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:20,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:20,571 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:22,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:22,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:22,573 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:23,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:23,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:23,818 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:24,574 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:25,702 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:25,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:26,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:27,740 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:27,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:28,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:28,870 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:29,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:29,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:30,580 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:32,582 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:33,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:33,128 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:33,972 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:34,584 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:35,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:35,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:36,586 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:37,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:37,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:38,587 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:39,157 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:39,194 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:57:39,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:39,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:40,588 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:41,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:41,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:42,590 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:43,298 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:43,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:44,246 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:44,591 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:46,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:46,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:46,594 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:48,576 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:48,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:48,595 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:49,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:49,530 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:57:50,596 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:50,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:50,638 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:52,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:52,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:52,598 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:53,896 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:53,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:54,420 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:54,599 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:55,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:55,946 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:56,601 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:57,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:57,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:58,602 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:57:59,541 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:00,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:00,019 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:00,603 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:02,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:02,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:02,605 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:04,606 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:04,661 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:05,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:05,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:06,608 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:07,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:07,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:07,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:07,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:08,610 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:09,488 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:09,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:09,733 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:10,612 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:11,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:11,547 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:12,614 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:14,615 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:14,886 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:15,734 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:58:15,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:15,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:16,618 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:17,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:18,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:18,619 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:19,532 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:58:19,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:20,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:20,149 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:20,621 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:22,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:22,105 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:22,325 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:22,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:22,622 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:24,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:24,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:24,624 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:25,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:26,626 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:26,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:26,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:28,627 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:28,847 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:28,870 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:30,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:30,629 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:31,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:31,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:32,630 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:33,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:33,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:34,632 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:35,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:35,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:35,440 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:36,634 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:37,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:37,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:37,525 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:37,560 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:38,635 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:39,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:39,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:40,243 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:40,636 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:41,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:42,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:42,638 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:44,185 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:44,211 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:44,639 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:45,258 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:46,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:46,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:46,642 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:48,643 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:49,141 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:58:49,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:49,535 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:58:49,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:50,327 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:50,645 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:52,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:52,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:52,646 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:52,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:52,924 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:54,648 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:55,144 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:55,162 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:55,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:56,650 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:57,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:57,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:58,651 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:58:59,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:59,528 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:00,451 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:00,652 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:01,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:01,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:02,654 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:03,917 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:03,938 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:04,655 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:05,500 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:06,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:06,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:06,656 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:07,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:07,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:08,228 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:08,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:08,658 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:10,392 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:10,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:10,536 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:10,660 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:12,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:12,624 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:12,662 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:14,663 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:14,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:14,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:15,582 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:16,665 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:17,015 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:17,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:18,667 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:19,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:19,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:19,537 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:59:20,668 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:20,710 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:21,358 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:21,377 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:22,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:22,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:22,670 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:24,671 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:25,199 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:59:25,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:25,729 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:25,743 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:26,673 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:27,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:27,914 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:28,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:30,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:30,119 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:30,676 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:30,822 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:32,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:32,267 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:32,678 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:34,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:34,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:34,679 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:35,830 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:36,681 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:36,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:36,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:37,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:37,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:38,683 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:38,872 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:38,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:40,684 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:40,900 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:41,042 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:41,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:42,686 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:43,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:43,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:44,688 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:45,417 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:45,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:45,994 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:46,691 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:47,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:47,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:48,693 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:49,540 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 20:59:49,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:49,809 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:50,694 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:51,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:51,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:51,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:52,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:52,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:52,696 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:54,159 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:54,172 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:54,698 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:56,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:56,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:56,400 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:56,700 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:58,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 20:59:59,832 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:00:00,701 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:00,703 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:00,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:01,553 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:02,705 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:02,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:02,990 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:04,707 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:05,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:05,049 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:06,663 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:06,709 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:07,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:07,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:07,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:07,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:08,710 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:09,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:09,174 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:10,712 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:11,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:11,322 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:11,798 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:12,714 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:13,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:13,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:14,715 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:15,730 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:15,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:16,719 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:16,981 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:17,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:17,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:18,720 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:19,542 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:00:19,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:19,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:20,722 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:22,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:22,064 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:22,072 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:22,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:22,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:22,724 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:24,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:24,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:24,726 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:26,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:26,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:26,728 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:27,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:28,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:28,684 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:28,729 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:30,731 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:30,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:30,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:32,270 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:32,733 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:34,735 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:35,138 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:00:35,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:35,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:36,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:37,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:37,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:37,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:37,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:37,412 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:38,738 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:39,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:39,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:40,740 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:41,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:41,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:42,497 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:42,742 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:43,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:43,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:44,744 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:46,121 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:46,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:46,747 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:47,574 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:48,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:48,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:48,749 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:49,545 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:00:50,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:50,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:50,751 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:52,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:52,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:52,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:52,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:52,753 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:52,845 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:54,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:54,469 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:54,754 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:56,497 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:56,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:56,756 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:00:57,888 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:58,552 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:58,563 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:58,758 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:00,759 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:01,722 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:01,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:02,760 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:02,914 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:03,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:03,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:04,762 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:05,932 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:05,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:06,763 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:07,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:07,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:08,016 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:08,765 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:09,930 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:01:09,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:09,979 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:10,766 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:12,014 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:12,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:12,768 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:13,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:14,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:14,077 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:14,770 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:16,773 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:17,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:17,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:18,165 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:18,774 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:19,547 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:01:20,100 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:20,134 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:20,776 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:22,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:22,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:22,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:22,287 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:22,778 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:23,203 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:24,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:24,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:24,780 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:26,781 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:27,570 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:27,598 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:28,290 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:28,782 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:29,677 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:29,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:30,784 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:31,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:31,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:32,786 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:33,383 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:33,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:33,807 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:34,787 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:35,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:35,839 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:36,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:37,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:37,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:37,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:37,949 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:38,487 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:38,790 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:39,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:40,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:40,792 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:42,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:42,045 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:42,793 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:43,590 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:44,795 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:46,019 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:01:46,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:46,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:46,799 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:48,077 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:48,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:48,660 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:48,800 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:49,550 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:01:50,802 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:51,301 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:51,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:52,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:52,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:52,804 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:53,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:53,403 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:54,464 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:54,805 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:55,438 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:55,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:58,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:58,382 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:58,807 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:01:59,591 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:00,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:00,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:00,809 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:02,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:02,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:02,810 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:04,689 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:04,747 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:04,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:04,812 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:06,814 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:06,990 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:07,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:07,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:07,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:08,815 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:09,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:09,226 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:09,794 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:10,817 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:11,343 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:11,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:12,818 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:13,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:13,528 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:14,820 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:15,529 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:15,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:15,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:16,823 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:17,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:17,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:19,552 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:02:20,554 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:21,590 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:02:21,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:21,653 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:22,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:22,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:22,825 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:24,829 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:24,917 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:24,938 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:26,060 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:26,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:27,000 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:28,831 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:29,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:29,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:30,833 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:31,068 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:31,078 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:31,092 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:32,835 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:33,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:33,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:34,837 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:36,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:36,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:36,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:36,839 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:37,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:37,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:38,840 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:39,374 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:39,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:40,842 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:41,235 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:41,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:41,517 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:42,844 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:43,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:43,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:44,845 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:46,250 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:46,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:46,816 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:46,849 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:48,847 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:48,851 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:48,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:49,555 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:02:50,853 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:50,908 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:50,921 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:51,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:52,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:52,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:52,855 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:52,955 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:52,993 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:54,856 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:55,051 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:55,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:56,458 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:56,858 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:02:58,861 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:00,608 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:03:00,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:00,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:00,861 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:01,510 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:02,728 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:02,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:02,863 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:04,810 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:04,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:04,865 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:06,533 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:06,866 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:06,877 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:06,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:07,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:07,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:08,868 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:08,949 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:08,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:10,870 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:11,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:11,051 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:11,544 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:12,872 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:13,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:13,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:14,874 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:15,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:15,176 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:16,623 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:16,877 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:18,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:18,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:18,879 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:19,560 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:03:20,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:20,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:20,881 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:21,781 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:22,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:22,107 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:22,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:22,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:22,883 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:24,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:24,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:24,885 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:26,851 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:26,887 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:27,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:27,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:28,888 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:29,425 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:29,440 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:30,890 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:31,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:31,495 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:31,887 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:32,892 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:34,894 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:35,424 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:03:35,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:35,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:36,896 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:37,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:37,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:37,317 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:38,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:38,797 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:40,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:40,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:40,898 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:42,886 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:42,887 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:42,900 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:42,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:44,925 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:44,942 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:46,903 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:48,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:48,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:48,338 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:49,560 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:03:50,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:50,402 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:50,904 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:52,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:52,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:52,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:52,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:52,906 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:54,331 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:55,566 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:55,593 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:56,908 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:57,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:57,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:58,910 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:03:59,587 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:59,696 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:59,709 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:01,737 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:01,747 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:03,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:03,782 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:04,820 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:04,912 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:05,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:05,811 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:06,914 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:07,107 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:07,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:07,832 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:07,842 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:08,915 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:10,299 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:10,916 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:13,207 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:04:13,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:13,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:15,301 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:15,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:15,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:17,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:17,493 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:19,562 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:04:19,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:19,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:20,563 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:21,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:21,729 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:22,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:22,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:23,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:23,834 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:25,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:25,982 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:26,264 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:28,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:28,094 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:28,920 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:30,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:30,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:32,079 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:32,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:32,377 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:34,496 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:34,525 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:36,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:36,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:36,923 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:37,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:37,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:37,286 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:38,802 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:38,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:40,925 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:40,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:40,994 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:42,567 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:43,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:43,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:44,927 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:46,968 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:04:46,969 DEBUG SenderThread:1257956 [sender.py:send():378] send: telemetry +2024-05-13 21:04:47,208 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:04:47,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:47,290 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:47,972 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:48,929 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:48,929 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/config.yaml +2024-05-13 21:04:49,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:49,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:49,565 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:04:50,930 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:04:51,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:51,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:52,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:52,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:53,263 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:53,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:53,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:55,470 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:55,480 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:57,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:57,516 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:58,265 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:59,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:59,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:01,571 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:01,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:03,266 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:03,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:03,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:05,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:05,753 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:06,935 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:07,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:07,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:07,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:07,794 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:08,273 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:09,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:09,841 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:11,896 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:11,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:13,274 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:13,984 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:14,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:14,937 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:16,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:16,088 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:18,934 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:19,568 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:05:20,127 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:05:20,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:20,212 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:20,939 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:22,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:22,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:22,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:22,274 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:23,982 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:23,983 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:05:23,984 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:05:23,984 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:05:23,984 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:05:24,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:24,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:24,941 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:24,941 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:05:26,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:26,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:28,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:28,476 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:28,943 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:28,990 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:30,520 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:30,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:30,945 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:32,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:32,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:34,262 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:34,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:34,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:36,705 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:36,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:37,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:37,108 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:38,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:38,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:39,319 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:40,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:40,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:42,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:43,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:44,877 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:44,949 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:45,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:45,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:47,276 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:47,309 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:49,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:49,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:49,569 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:05:50,570 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:52,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:52,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:53,491 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:05:53,560 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:53,580 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:54,952 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:05:55,666 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:55,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:56,392 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:57,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:57,824 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:59,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:05:59,650 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:05:59,650 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:05:59,650 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:05:59,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:59,954 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:05:59,958 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:00,954 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:01,651 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:02,102 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:02,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:04,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:04,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:04,957 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:06,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:06,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:07,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:07,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:07,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:08,520 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:08,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:08,958 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:10,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:10,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:10,966 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:12,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:12,800 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:13,061 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:14,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:15,004 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:17,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:17,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:18,062 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:19,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:19,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:19,571 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:06:20,969 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:21,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:21,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:22,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:22,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:23,334 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:23,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:23,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:27,508 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:06:27,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:27,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:28,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:29,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:29,623 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:31,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:31,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:32,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:06:32,130 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:06:32,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:06:32,130 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:06:32,972 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:32,972 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:06:33,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:33,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:34,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:35,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:35,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:37,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:37,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:37,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:37,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:39,874 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:39,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:39,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:40,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:42,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:42,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:44,128 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:44,150 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:44,976 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:45,608 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:46,171 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:46,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:48,204 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:48,219 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:48,977 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:49,573 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:06:50,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:50,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:50,917 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:50,978 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:51,886 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:06:51,887 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:06:51,887 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:06:51,888 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:06:51,978 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:06:52,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:52,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:52,288 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:52,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:52,978 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:06:55,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:55,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:56,325 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:57,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:57,511 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:01,326 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:01,442 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:07:01,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:01,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:03,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:03,512 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:05,537 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:05,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:06,328 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:07,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:07,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:07,570 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:07,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:09,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:09,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:10,984 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:11,641 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:11,659 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:12,300 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:13,683 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:13,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:15,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:15,750 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:17,781 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:17,795 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:18,088 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:18,986 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:19,576 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:07:20,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:20,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:22,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:22,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:22,938 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:22,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:22,988 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:23,497 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:24,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:24,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:26,989 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:27,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:27,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:28,989 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:29,041 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:07:29,042 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:07:29,042 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:07:29,043 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:07:29,990 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:07:30,190 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:30,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:30,991 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:32,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:32,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:32,992 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:34,004 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:36,112 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:07:36,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:36,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:37,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:37,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:38,683 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:38,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:39,273 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:40,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:40,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:42,827 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:42,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:44,273 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:44,905 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:44,921 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:47,002 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:47,014 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:48,998 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:49,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:49,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:49,580 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:07:49,581 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:51,186 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:51,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:52,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:52,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:53,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:53,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:55,251 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:55,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:55,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:57,000 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:07:57,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:57,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:59,697 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:59,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:00,879 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:01,846 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:01,874 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:03,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:03,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:03,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:05,728 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:08:05,729 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:08:05,729 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:08:05,730 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:08:06,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:08:06,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:06,146 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:06,730 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:07,003 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:07,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:07,109 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:10,271 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:08:10,344 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:10,364 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:11,004 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:12,398 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:12,440 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:12,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:13,006 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:14,524 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:14,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:16,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:16,665 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:17,398 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:18,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:18,729 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:19,581 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:08:20,774 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:20,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:22,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:22,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:22,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:22,841 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:23,278 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:24,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:24,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:26,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:26,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:27,010 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:28,525 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:28,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:28,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:31,015 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:31,030 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:33,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:33,072 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:33,526 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:35,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:35,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:37,012 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:37,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:37,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:37,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:37,265 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:39,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:39,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:39,384 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:41,523 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:08:41,525 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:08:41,525 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:08:41,525 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:08:42,014 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:08:43,014 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:43,534 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:08:43,611 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:43,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:44,526 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:45,709 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:45,745 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:47,015 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:47,832 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:47,848 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:49,583 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:08:49,584 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:49,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:49,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:51,022 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:52,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:52,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:52,109 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:52,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:53,023 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:08:54,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:54,081 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:55,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:56,121 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:56,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:58,177 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:58,188 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:00,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:00,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:00,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:02,305 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:02,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:03,026 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:04,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:04,388 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:05,901 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:06,429 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:06,443 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:07,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:07,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:08,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:08,506 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:10,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:10,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:11,260 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:12,594 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:12,606 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:14,094 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:09:14,095 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:09:14,095 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:09:14,096 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:09:15,029 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:15,030 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:09:16,329 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:09:16,802 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:16,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:17,096 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:18,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:18,959 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:19,586 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:09:21,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:21,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:22,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:22,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:22,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:23,032 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:23,128 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:23,150 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:25,198 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:25,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:27,258 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:27,353 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:27,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:29,034 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:29,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:29,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:31,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:31,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:32,374 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:33,036 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:33,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:33,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:35,038 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:35,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:35,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:35,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:09:35,999 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:09:36,000 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:09:36,000 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:09:36,038 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:09:37,039 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:37,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:37,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:37,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:37,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:38,243 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:39,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:39,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:41,972 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:41,984 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:43,243 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:44,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:44,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:46,047 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:46,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:48,244 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:49,588 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:09:50,051 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:09:50,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:50,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:52,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:52,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:53,404 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:53,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:54,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:55,044 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:09:55,485 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:55,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:57,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:57,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:59,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:59,636 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:59,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:01,046 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:01,704 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:01,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:03,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:03,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:04,467 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:05,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:05,815 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:07,048 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:07,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:07,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:07,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:07,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:09,638 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:11,049 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:11,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:11,161 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:12,473 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:10:12,475 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:10:12,475 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:10:12,475 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:10:13,051 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:13,051 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:10:13,212 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:13,234 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:15,052 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:15,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:15,278 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:15,526 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:17,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:17,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:19,591 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:10:20,593 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:20,778 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:20,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:22,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:22,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:22,820 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:22,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:26,339 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:26,771 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:10:26,794 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:26,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:28,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:28,836 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:30,857 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:30,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:31,341 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:32,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:32,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:33,056 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:34,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:34,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:36,949 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:37,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:37,110 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:38,161 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:38,190 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:40,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:40,252 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:42,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:42,305 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:42,335 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:43,059 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:44,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:44,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:46,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:46,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:47,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:49,061 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:49,593 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:10:49,594 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:49,613 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:51,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:51,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:52,085 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:10:52,086 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:10:52,086 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:10:52,086 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:10:52,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:52,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:53,062 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:53,062 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:10:53,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:53,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:53,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:55,758 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:55,771 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:57,064 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:10:58,605 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:58,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:59,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:59,065 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:03,145 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:11:03,185 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:03,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:03,606 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:05,278 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:05,302 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:07,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:07,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:07,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:07,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:09,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:09,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:09,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:11,594 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:11,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:13,069 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:13,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:13,773 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:14,478 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:15,857 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:15,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:17,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:18,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:19,479 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:19,595 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:11:20,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:20,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:22,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:22,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:22,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:22,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:23,072 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:24,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:24,522 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:24,613 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:26,590 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:26,622 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:28,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:28,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:29,450 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:11:29,451 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:11:29,451 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:11:29,452 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:11:30,074 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:11:30,452 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:30,746 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:30,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:31,074 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:33,369 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:33,397 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:35,075 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:36,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:37,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:37,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:37,438 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:11:37,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:37,476 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:39,077 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:39,511 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:39,527 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:41,078 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:41,555 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:41,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:41,985 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:43,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:43,651 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:45,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:45,795 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:46,985 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:47,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:47,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:49,081 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:11:49,598 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:11:50,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:50,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:52,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:52,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:52,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:52,238 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:52,251 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:54,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:54,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:56,403 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:56,439 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:57,238 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:58,479 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:58,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:00,527 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:00,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:00,592 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:00,593 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:12:00,593 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:12:00,593 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:12:01,084 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:01,084 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:12:02,566 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:02,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:02,594 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:03,085 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:04,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:04,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:06,651 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:06,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:07,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:07,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:07,801 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:09,086 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:10,768 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:12:10,824 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:10,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:12,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:12,886 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:13,088 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:13,582 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:14,908 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:14,925 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:16,946 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:16,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:17,090 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:18,609 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:18,991 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:19,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:19,092 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:19,601 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:12:20,527 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:20,528 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:12:20,528 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:12:20,528 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:12:21,093 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:21,094 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:12:22,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:22,111 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:22,319 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:22,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:23,094 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:24,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:24,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:24,392 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:26,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:26,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:28,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:28,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:29,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:30,515 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:30,525 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:33,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:33,863 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:34,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:35,899 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:35,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:37,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:37,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:37,951 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:37,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:39,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:40,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:40,024 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:41,100 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:42,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:42,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:45,025 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:45,992 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:12:46,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:46,050 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:48,102 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:48,119 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:49,101 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:49,603 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:12:50,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:50,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:50,605 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:52,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:52,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:52,253 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:52,266 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:53,103 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:54,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:54,329 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:55,815 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:56,366 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:56,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:57,104 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:58,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:58,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:58,630 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:58,631 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:12:58,632 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:12:58,632 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:12:59,105 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:12:59,106 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:13:00,459 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:00,481 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:01,106 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:01,617 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:02,506 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:02,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:04,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:04,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:06,588 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:06,606 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:06,618 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:07,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:07,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:08,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:08,641 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:10,661 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:10,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:12,341 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:12,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:12,728 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:14,750 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:14,762 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:17,110 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:18,320 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:18,864 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:13:18,891 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:18,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:19,605 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:13:20,930 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:20,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:22,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:22,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:22,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:22,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:23,342 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:25,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:25,014 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:27,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:27,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:27,113 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:28,829 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:29,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:29,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:31,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:31,337 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:33,114 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:33,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:33,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:34,643 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:35,456 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:35,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:35,807 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:13:35,809 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:13:35,809 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:13:35,809 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:13:36,115 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:13:37,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:37,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:37,116 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:37,511 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:37,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:39,117 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:39,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:39,580 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:39,726 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:41,623 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:41,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:43,119 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:43,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:43,688 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:44,901 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:45,720 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:45,738 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:47,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:47,795 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:49,607 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:13:50,608 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:51,568 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:13:52,051 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:52,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:52,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:52,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:54,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:54,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:55,123 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:13:56,333 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:56,367 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:56,373 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:58,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:58,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:00,614 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:00,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:01,373 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:02,704 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:02,737 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:04,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:04,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:05,127 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:06,818 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:06,914 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:06,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:07,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:07,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:08,977 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:09,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:10,043 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:14:10,044 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:14:10,044 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:14:10,045 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:14:10,128 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:14:11,128 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:11,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:11,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:12,045 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:13,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:13,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:15,129 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:15,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:15,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:17,279 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:17,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:17,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:19,131 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:19,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:19,581 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:19,609 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:14:21,133 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:21,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:21,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:22,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:22,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:23,271 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:25,689 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:14:25,740 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:25,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:27,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:27,856 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:28,273 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:29,884 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:29,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:31,136 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:31,921 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:31,942 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:33,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:33,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:34,254 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:36,002 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:36,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:37,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:37,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:38,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:38,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:39,507 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:40,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:40,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:42,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:42,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:42,898 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:14:42,899 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:14:42,899 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:14:42,899 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:14:43,143 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:43,143 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:14:44,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:44,346 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:44,900 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:45,144 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:46,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:46,470 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:48,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:48,599 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:49,612 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:14:50,371 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:50,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:50,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:51,146 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:52,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:52,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:52,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:52,937 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:55,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:55,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:55,585 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:57,148 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:14:59,121 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:14:59,186 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:59,212 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:01,119 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:01,149 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:01,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:01,274 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:03,150 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:03,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:03,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:03,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:15:03,650 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:15:03,650 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:15:03,651 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:15:04,150 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:15:05,150 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:05,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:05,395 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:06,651 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:07,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:07,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:07,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:07,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:09,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:09,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:11,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:11,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:12,327 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:13,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:13,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:15,641 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:15,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:17,328 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:17,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:17,693 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:19,615 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:15:19,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:19,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:21,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:21,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:22,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:22,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:23,157 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:23,241 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:24,061 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:24,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:26,206 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:26,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:28,241 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:28,319 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:28,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:31,159 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:32,025 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:15:32,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:32,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:33,684 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:34,568 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:34,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:36,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:36,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:37,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:37,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:37,161 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:38,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:38,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:39,074 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:40,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:40,800 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:41,162 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:42,065 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:15:42,067 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:15:42,067 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:15:42,067 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:15:42,162 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:15:42,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:42,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:43,163 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:44,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:45,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:45,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:45,163 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:15:47,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:47,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:49,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:49,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:49,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:49,617 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:15:51,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:51,388 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:52,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:52,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:53,464 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:53,493 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:54,267 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:55,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:55,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:57,705 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:57,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:59,268 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:59,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:59,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:01,836 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:01,846 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:03,168 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:05,078 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:05,993 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:16:06,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:06,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:07,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:07,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:08,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:08,077 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:10,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:10,153 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:10,893 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:11,171 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:12,344 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:12,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:14,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:14,545 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:16,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:16,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:16,690 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:17,173 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:18,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:18,807 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:19,475 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:16:19,476 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:16:19,477 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:16:19,477 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:16:19,619 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:16:20,174 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:16:20,935 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:20,962 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:21,174 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:21,621 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:22,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:22,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:23,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:23,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:23,175 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:25,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:25,185 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:25,204 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:27,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:27,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:27,419 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:29,427 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:29,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:31,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:31,563 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:32,420 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:33,631 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:33,661 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:35,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:35,721 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:37,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:37,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:38,255 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:39,181 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:39,770 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:16:39,809 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:39,819 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:41,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:41,891 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:43,256 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:43,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:43,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:45,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:46,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:48,118 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:48,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:48,944 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:49,183 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:49,622 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:16:50,302 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:50,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:52,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:52,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:52,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:52,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:53,332 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:16:53,333 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:16:53,334 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:16:53,334 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:16:54,186 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:16:54,334 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:54,576 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:54,603 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:55,186 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:56,704 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:56,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:58,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:58,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:59,187 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:16:59,670 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:00,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:00,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:03,060 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:03,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:03,189 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:05,152 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:05,180 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:05,191 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:05,339 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:07,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:07,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:07,252 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:07,265 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:09,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:09,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:11,313 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:13,444 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:17:13,484 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:13,496 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:15,194 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:15,603 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:15,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:16,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:17,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:17,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:19,625 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:17:19,783 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:19,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:21,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:21,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:22,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:22,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:23,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:23,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:25,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:25,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:26,161 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:17:26,792 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:17:26,793 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:17:26,793 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:26,794 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:17:27,197 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:27,198 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:17:28,002 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:28,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:29,198 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:30,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:30,066 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:31,795 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:33,252 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:33,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:35,200 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:35,337 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:35,373 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:37,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:37,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:37,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:37,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:37,427 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:39,202 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:39,448 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:39,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:41,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:41,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:42,432 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:43,203 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:43,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:43,555 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:46,864 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:17:46,865 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:17:46,865 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:17:46,866 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:17:47,204 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:47,205 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:17:47,464 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:17:47,497 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:47,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:47,866 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:49,205 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:17:49,627 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:17:50,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:50,845 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:52,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:52,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:52,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:52,926 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:53,292 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:54,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:55,002 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:57,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:57,099 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:58,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:00,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:00,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:02,631 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:02,662 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:03,209 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:03,938 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:04,753 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:04,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:06,875 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:06,901 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:07,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:18:07,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:18:07,210 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:08,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:08,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:09,281 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:10,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:11,021 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:13,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:13,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:13,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:15,126 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:15,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:15,193 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:15,213 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:17,170 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:17,186 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:19,629 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:18:20,427 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:20,439 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:20,630 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:22,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:18:22,116 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:18:24,344 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:18:24,370 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:24,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:26,264 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:26,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:26,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:28,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:28,456 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:30,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:30,495 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:31,265 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:33,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:33,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:35,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:35,782 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:36,266 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:37,114 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:18:37,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:18:37,220 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:37,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:37,816 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:39,230 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:39,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:39,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:41,239 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:41,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:41,927 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:42,071 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:42,576 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 21:18:42,577 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 21:18:42,890 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:18:42,938 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:18:42,940 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav +2024-05-13 21:18:43,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table +2024-05-13 21:18:43,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media +2024-05-13 21:18:43,752 INFO wandb-upload_0:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmp21h4u61y +2024-05-13 21:18:43,789 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/p8xnqiah-media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json +2024-05-13 21:18:44,300 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ahahvwe5-media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav +2024-05-13 21:18:44,412 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep127 - {'id': 'QXJ0aWZhY3Q6ODM0NzY0OTc0', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI0NTA0OA==', 'latestArtifact': None}} +2024-05-13 21:18:44,412 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,412 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json with policy now +2024-05-13 21:18:44,413 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:18:44,413 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:18:44,413 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:18:44,414 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav with policy now +2024-05-13 21:18:44,414 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav with policy now +2024-05-13 21:18:44,414 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav with policy now +2024-05-13 21:18:44,414 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav with policy now +2024-05-13 21:18:44,414 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav with policy now +2024-05-13 21:18:44,415 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,415 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav with policy now +2024-05-13 21:18:44,415 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:18:44,415 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav with policy now +2024-05-13 21:18:44,478 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/4qc266nw-media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav +2024-05-13 21:18:44,480 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ewag4pgb-media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav +2024-05-13 21:18:44,509 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/3q9cism8-media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav +2024-05-13 21:18:44,536 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/iqs57vqj-media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav +2024-05-13 21:18:44,541 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/mglkmosw-media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav +2024-05-13 21:18:44,550 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/wte40eui-media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav +2024-05-13 21:18:44,886 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/uvsggul8-media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json +2024-05-13 21:18:45,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:45,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:18:45,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:45,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:45,469 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/1gpdin7d-media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav +2024-05-13 21:18:45,564 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/zjlxcjkp-media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav +2024-05-13 21:18:45,587 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ajegc54h-media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav +2024-05-13 21:18:45,611 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/z5ylnkvx-media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav +2024-05-13 21:18:45,913 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/95d2petp-media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav +2024-05-13 21:18:46,521 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/fkmf1x1x-media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav +2024-05-13 21:18:46,606 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/w8uqvnxv-media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav +2024-05-13 21:18:47,243 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:18:47,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:47,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:47,475 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:49,416 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:49,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:49,631 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:18:51,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:51,466 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:52,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:18:52,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:18:53,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:53,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:53,586 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:55,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:55,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:58,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:59,452 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:18:59,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:59,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:01,997 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:02,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:03,325 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:04,119 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:04,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:06,226 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:06,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:07,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:19:07,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:19:07,253 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:08,278 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:08,288 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:10,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:10,343 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:11,824 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:12,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:12,404 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:13,642 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:19:13,643 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:19:13,644 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:19:13,644 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:19:14,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:19:14,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:14,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:15,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:16,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:16,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:17,645 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:18,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:18,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:19,633 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:19:20,702 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:20,737 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:21,259 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:22,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:19:22,116 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:19:22,877 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:22,905 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:22,960 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:24,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:25,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:25,261 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:27,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:27,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:27,262 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:28,811 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:29,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:29,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:29,263 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:31,860 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:19:31,886 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:33,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:33,811 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:35,420 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:35,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:37,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:19:37,116 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:19:37,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:37,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:39,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:39,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:39,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:41,862 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:41,896 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:44,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:44,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:44,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:45,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:19:45,177 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:19:45,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:19:45,178 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:19:45,271 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:19:46,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:46,129 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:47,272 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:48,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:48,293 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:49,636 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:19:49,637 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:49,755 WARNING FileStreamThread:1257956 [file_stream.py:request_with_retry():674] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/sanchit-gandhi/parler-speech/qaoje1x9/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/sanchit-gandhi/parler-speech/qaoje1x9/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"train/loss": 4.445040225982666, "train/time": 819.1428015232086, "train/epoch": 0.984251968503937, "train/learning_rate": 4e-05, "_timestamp": 1715627866.8630917, "_runtime": 1497.8430137634277, "_step": 125}']}, 'wandb-history.jsonl': {'offset': 24, 'content': ['{"train/loss": 4.445040225982666, "train/time": 819.1428015232086, "train/epoch": 0.984251968503937, "train/learning_rate": 4e-05, "_timestamp": 1715627866.8630917, "_runtime": 1497.8430137634277, "_step": 125}']}, 'wandb-events.jsonl': {'offset': 50, 'content': ['{"system.proc.memory.availableMB": 406600.59, "system.memory": 21.15, "system.proc.memory.rssMB": 7339.59, "system.proc.memory.percent": 1.42, "system.network.sent": 215219120.0, "system.network.recv": 139749001.07, "system.cpu": 0.79, "system.cpu.0.cpu_percent": 11.47, "system.cpu.1.cpu_percent": 4.03, "system.cpu.2.cpu_percent": 37.67, "system.cpu.3.cpu_percent": 6.23, "system.cpu.4.cpu_percent": 8.89, "system.cpu.5.cpu_percent": 1.69, "system.cpu.6.cpu_percent": 14.29, "system.cpu.7.cpu_percent": 0.23, "system.cpu.8.cpu_percent": 10.91, "system.cpu.9.cpu_percent": 7.67, "system.cpu.10.cpu_percent": 0.27, "system.cpu.11.cpu_percent": 8.63, "system.cpu.12.cpu_percent": 1.03, "system.cpu.13.cpu_percent": 0.03, "system.cpu.14.cpu_percent": 0.0, "system.cpu.15.cpu_percent": 0.2, "system.cpu.16.cpu_percent": 0.03, "system.cpu.17.cpu_percent": 0.17, "system.cpu.18.cpu_percent": 0.03, "system.cpu.19.cpu_percent": 0.13, "system.cpu.20.cpu_percent": 7.62, "system.cpu.21.cpu_percent": 6.03, "system.cpu.22.cpu_percent": 0.43, "system.cpu.23.cpu_percent": 3.47, "system.cpu.24.cpu_percent": 0.07, "system.cpu.25.cpu_percent": 62.72, "system.cpu.26.cpu_percent": 0.23, "system.cpu.27.cpu_percent": 0.1, "system.cpu.28.cpu_percent": 5.06, "system.cpu.29.cpu_percent": 1.37, "system.cpu.30.cpu_percent": 0.1, "system.cpu.31.cpu_percent": 0.17, "system.cpu.32.cpu_percent": 58.22, "system.cpu.33.cpu_percent": 9.39, "system.cpu.34.cpu_percent": 1.1, "system.cpu.35.cpu_percent": 0.03, "system.cpu.36.cpu_percent": 5.53, "system.cpu.37.cpu_percent": 4.13, "system.cpu.38.cpu_percent": 0.47, "system.cpu.39.cpu_percent": 0.23, "system.cpu.40.cpu_percent": 15.57, "system.cpu.41.cpu_percent": 8.1, "system.cpu.42.cpu_percent": 0.7, "system.cpu.43.cpu_percent": 0.63, "system.cpu.44.cpu_percent": 2.65, "system.cpu.45.cpu_percent": 8.46, "system.cpu.46.cpu_percent": 2.2, "system.cpu.47.cpu_percent": 1.45, "system.cpu.48.cpu_percent": 44.57, "system.cpu.49.cpu_percent": 0.03, "system.cpu.50.cpu_percent": 34.19, "system.cpu.51.cpu_percent": 26.78, "system.cpu.52.cpu_percent": 20.47, "system.cpu.53.cpu_percent": 22.31, "system.cpu.54.cpu_percent": 2.37, "system.cpu.55.cpu_percent": 1.59, "system.cpu.56.cpu_percent": 0.87, "system.cpu.57.cpu_percent": 31.11, "system.cpu.58.cpu_percent": 0.73, "system.cpu.59.cpu_percent": 3.45, "system.cpu.60.cpu_percent": 7.25, "system.cpu.61.cpu_percent": 0.07, "system.cpu.62.cpu_percent": 0.1, "system.cpu.63.cpu_percent": 2.47, "system.cpu.64.cpu_percent": 0.13, "system.cpu.65.cpu_percent": 3.37, "system.cpu.66.cpu_percent": 8.77, "system.cpu.67.cpu_percent": 3.01, "system.cpu.68.cpu_percent": 0.57, "system.cpu.69.cpu_percent": 1.84, "system.cpu.70.cpu_percent": 2.23, "system.cpu.71.cpu_percent": 0.87, "system.cpu.72.cpu_percent": 0.4, "system.cpu.73.cpu_percent": 12.17, "system.cpu.74.cpu_percent": 1.23, "system.cpu.75.cpu_percent": 1.69, "system.cpu.76.cpu_percent": 0.23, "system.cpu.77.cpu_percent": 0.33, "system.cpu.78.cpu_percent": 0.13, "system.cpu.79.cpu_percent": 0.2, "system.cpu.80.cpu_percent": 99.53, "system.cpu.81.cpu_percent": 0.1, "system.cpu.82.cpu_percent": 0.07, "system.cpu.83.cpu_percent": 0.07, "system.cpu.84.cpu_percent": 0.0, "system.cpu.85.cpu_percent": 1.6, "system.cpu.86.cpu_percent": 0.1, "system.cpu.87.cpu_percent": 0.07, "system.cpu.88.cpu_percent": 36.33, "system.cpu.89.cpu_percent": 0.13, "system.cpu.90.cpu_percent": 0.1, "system.cpu.91.cpu_percent": 0.17, "system.cpu.92.cpu_percent": 0.3, "system.cpu.93.cpu_percent": 4.84, "system.cpu.94.cpu_percent": 0.87, "system.cpu.95.cpu_percent": 1.76, "system.cpu.96.cpu_percent": 0.2, "system.cpu.97.cpu_percent": 0.07, "system.cpu.98.cpu_percent": 0.3, "system.cpu.99.cpu_percent": 0.2, "system.cpu.100.cpu_percent": 3.26, "system.cpu.101.cpu_percent": 2.1, "system.cpu.102.cpu_percent": 0.33, "system.cpu.103.cpu_percent": 0.64, "system.cpu.104.cpu_percent": 2.16, "system.cpu.105.cpu_percent": 4.71, "system.cpu.106.cpu_percent": 0.8, "system.cpu.107.cpu_percent": 0.03, "system.cpu.108.cpu_percent": 0.13, "system.cpu.109.cpu_percent": 0.5, "system.cpu.110.cpu_percent": 0.23, "system.cpu.111.cpu_percent": 2.47, "system.cpu.112.cpu_percent": 0.13, "system.cpu.113.cpu_percent": 0.03, "system.cpu.114.cpu_percent": 0.1, "system.cpu.115.cpu_percent": 0.0, "system.cpu.116.cpu_percent": 0.23, "system.cpu.117.cpu_percent": 0.03, "system.cpu.118.cpu_percent": 3.53, "system.cpu.119.cpu_percent": 0.1, "system.cpu.120.cpu_percent": 0.8, "system.cpu.121.cpu_percent": 1.03, "system.cpu.122.cpu_percent": 0.8, "system.cpu.123.cpu_percent": 0.57, "system.cpu.124.cpu_percent": 5.35, "system.cpu.125.cpu_percent": 0.37, "system.cpu.126.cpu_percent": 0.13, "system.cpu.127.cpu_percent": 0.1, "system.proc.cpu.threads": 78, "system.disk.\\\\.usagePercent": 99.7, "system.disk.\\\\.usageGB": 1663.51, "system.disk.in": 65.27, "system.disk.out": 13959.2, "system.gpu.0.memoryAllocated": 85.43, "system.gpu.1.memoryAllocated": 68.86, "system.gpu.process.1.memoryAllocated": 68.86, "system.gpu.2.memoryAllocated": 59.42, "system.gpu.3.memoryAllocated": 4.69, "system.gpu.4.memoryAllocated": 42.38, "system.gpu.0.memoryAllocatedBytes": 73387802624.0, "system.gpu.1.memoryAllocatedBytes": 59154497536.0, "system.gpu.process.1.memoryAllocatedBytes": 59154497536.0, "system.gpu.2.memoryAllocatedBytes": 51038388224.0, "system.gpu.3.memoryAllocatedBytes": 201588736.0, "system.gpu.4.memoryAllocatedBytes": 36444866696.53, "system.gpu.0.memory": 0.0, "system.gpu.1.memory": 30.2, "system.gpu.process.1.memory": 30.2, "system.gpu.2.memory": 48.13, "system.gpu.3.memory": 0.0, "system.gpu.4.memory": 34.67, "system.gpu.0.gpu": 0.0, "system.gpu.1.gpu": 66.8, "system.gpu.process.1.gpu": 66.8, "system.gpu.2.gpu": 96.87, "system.gpu.3.gpu": 0.0, "system.gpu.4.gpu": 74.93, "system.gpu.0.temp": 70.87, "system.gpu.1.temp": 77.07, "system.gpu.process.1.temp": 77.07, "system.gpu.2.temp": 77.73, "system.gpu.3.temp": 40.0, "system.gpu.4.temp": 77.13, "_wandb": true, "_timestamp": 1715627929.628074, "_runtime": 1560.607996}']}, 'output.log': {'offset': 921, 'content': ['ERROR 2024-05-13T19:18:39.230068 31,7,r_norm_epsilon": 1e-06,\n', 'ERROR 2024-05-13T19:18:39.230068 3246,\n', 'ERROR 2024-05-13T19:18:39.230068 3253,\n', 'ERROR 2024-05-13T19:18:39.230068 3268,\n', 'ERROR 2024-05-13T19:18:39.230068 3536,\n', 'ERROR 2024-05-13T19:18:39.230068 3846,\n', 'ERROR 2024-05-13T19:18:39.230068 3961,\n', 'ERROR 2024-05-13T19:18:39.230068 4183,\n', 'ERROR 2024-05-13T19:18:39.230068 4667,\n', 'ERROR 2024-05-13T19:18:39.230068 6585,\n', 'ERROR 2024-05-13T19:18:39.230068 6647,\n', 'ERROR 2024-05-13T19:18:39.230068 7273,\n', 'ERROR 2024-05-13T19:18:39.230068 9061,\n', 'ERROR 2024-05-13T19:18:39.230068 9383,\n', 'ERROR 2024-05-13T19:18:39.230068 10428,\n', 'ERROR 2024-05-13T19:18:39.230068 10929,\n', 'ERROR 2024-05-13T19:18:39.230068 11938,\n', 'ERROR 2024-05-13T19:18:39.230068 12033,\n', 'ERROR 2024-05-13T19:18:39.230068 12331,\n', 'ERROR 2024-05-13T19:18:39.230068 12562,\n', 'ERROR 2024-05-13T19:18:39.230068 13793,\n', 'ERROR 2024-05-13T19:18:39.230068 14157,\n', 'ERROR 2024-05-13T19:18:39.230068 14635,\n', 'ERROR 2024-05-13T19:18:39.230068 15265,\n', 'ERROR 2024-05-13T19:18:39.230068 15618,\n', 'ERROR 2024-05-13T19:18:39.230068 16553,\n', 'ERROR 2024-05-13T19:18:39.230068 16604,\n', 'ERROR 2024-05-13T19:18:39.230068 18362,\n', 'ERROR 2024-05-13T19:18:39.230068 18956,\n', 'ERROR 2024-05-13T19:18:39.230068 20075,\n', 'ERROR 2024-05-13T19:18:39.230068 21675,\n', 'ERROR 2024-05-13T19:18:39.230068 22520,\n', 'ERROR 2024-05-13T19:18:39.230068 26130,\n', 'ERROR 2024-05-13T19:18:39.230068 26161,\n', 'ERROR 2024-05-13T19:18:39.230068 26435,\n', 'ERROR 2024-05-13T19:18:39.230068 28279,\n', 'ERROR 2024-05-13T19:18:39.230068 29464,\n', 'ERROR 2024-05-13T19:18:39.230068 31650,\n', 'ERROR 2024-05-13T19:18:39.230068 32302,\n', 'ERROR 2024-05-13T19:18:39.230068 32470,\n', 'ERROR 2024-05-13T19:18:39.230068 36865,\n', 'ERROR 2024-05-13T19:18:39.230068 42863,\n', 'ERROR 2024-05-13T19:18:39.230068 47425,\n', 'ERROR 2024-05-13T19:18:39.230068 49870,\n', 'ERROR 2024-05-13T19:18:39.230068 50254,\n', 'ERROR 2024-05-13T19:18:39.230068 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 50360,\n', 'ERROR 2024-05-13T19:18:39.230068 50361,\n', 'ERROR 2024-05-13T19:18:39.230068 50362\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "torch_dtype": "float32",\n', 'ERROR 2024-05-13T19:18:39.230068 "transformers_version": "4.41.0.dev0",\n', 'ERROR 2024-05-13T19:18:39.230068 "use_cache": true,\n', 'ERROR 2024-05-13T19:18:39.230068 "use_weighted_layer_sum": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "vocab_size": 51865\n', 'ERROR 2024-05-13T19:18:39.230068 }\n', 'ERROR 2024-05-13T19:18:39.230068 loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json\n', 'ERROR 2024-05-13T19:18:39.230068 Model config WhisperConfig {\n', 'ERROR 2024-05-13T19:18:39.230068 "_name_or_path": "distil-whisper/distil-large-v2",\n', 'ERROR 2024-05-13T19:18:39.230068 "activation_dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "activation_function": "gelu",\n', 'ERROR 2024-05-13T19:18:39.230068 "apply_spec_augment": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "architectures": [\n', 'ERROR 2024-05-13T19:18:39.230068 "WhisperForConditionalGeneration"\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "attention_dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "begin_suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 220,\n', 'ERROR 2024-05-13T19:18:39.230068 50257\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "bos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "classifier_proj_size": 256,\n', 'ERROR 2024-05-13T19:18:39.230068 "d_model": 1280,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_attention_heads": 20,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_ffn_dim": 5120,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_layerdrop": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_layers": 2,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_start_token_id": 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 "dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_attention_heads": 20,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_ffn_dim": 5120,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_layerdrop": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_layers": 32,\n', 'ERROR 2024-05-13T19:18:39.230068 "eos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "forced_decoder_ids": [\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 50259\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 50359\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 3,\n', 'ERROR 2024-05-13T19:18:39.230068 50363\n', 'ERROR 2024-05-13T19:18:39.230068 ]\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "init_std": 0.02,\n', 'ERROR 2024-05-13T19:18:39.230068 "is_encoder_decoder": true,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_length": 10,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_min_masks": 0,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_prob": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_length": 10,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_min_masks": 2,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_prob": 0.05,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_length": 448,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_source_positions": 1500,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_target_positions": 448,\n', 'ERROR 2024-05-13T19:18:39.230068 "median_filter_width": 7,\n', 'ERROR 2024-05-13T19:18:39.230068 "model_type": "whisper",\n', 'ERROR 2024-05-13T19:18:39.230068 "num_hidden_layers": 32,\n', 'ERROR 2024-05-13T19:18:39.230068 "num_mel_bins": 80,\n', 'ERROR 2024-05-13T19:18:39.230068 "pad_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "scale_embedding": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 7,\n', 'ERROR 2024-05-13T19:18:39.230068 8,\n', 'ERROR 2024-05-13T19:18:39.230068 9,\n', 'ERROR 2024-05-13T19:18:39.230068 10,\n', 'ERROR 2024-05-13T19:18:39.230068 14,\n', 'ERROR 2024-05-13T19:18:39.230068 25,\n', 'ERROR 2024-05-13T19:18:39.230068 26,\n', 'ERROR 2024-05-13T19:18:39.230068 27,\n', 'ERROR 2024-05-13T19:18:39.230068 28,\n', 'ERROR 2024-05-13T19:18:39.230068 29,\n', 'ERROR 2024-05-13T19:18:39.230068 31,7,r_norm_epsilon": 1e-06,\n', 'ERROR 2024-05-13T19:18:39.230068 58,\n', 'ERROR 2024-05-13T19:18:39.230068 59,\n', 'ERROR 2024-05-13T19:18:39.230068 60,\n', 'ERROR 2024-05-13T19:18:39.230068 61,\n', 'ERROR 2024-05-13T19:18:39.230068 62,\n', 'ERROR 2024-05-13T19:18:39.230068 63,\n', 'ERROR 2024-05-13T19:18:39.230068 90,\n', 'ERROR 2024-05-13T19:18:39.230068 91,\n', 'ERROR 2024-05-13T19:18:39.230068 92,\n', 'ERROR 2024-05-13T19:18:39.230068 93,\n', 'ERROR 2024-05-13T19:18:39.230068 359,\n', 'ERROR 2024-05-13T19:18:39.230068 503,\n', 'ERROR 2024-05-13T19:18:39.230068 522,\n', 'ERROR 2024-05-13T19:18:39.230068 542,\n', 'ERROR 2024-05-13T19:18:39.230068 873,\n', 'ERROR 2024-05-13T19:18:39.230068 893,\n', 'ERROR 2024-05-13T19:18:39.230068 902,\n', 'ERROR 2024-05-13T19:18:39.230068 918,\n', 'ERROR 2024-05-13T19:18:39.230068 922,\n', 'ERROR 2024-05-13T19:18:39.230068 931,\n', 'ERROR 2024-05-13T19:18:39.230068 1350,\n', 'ERROR 2024-05-13T19:18:39.230068 1853,\n', 'ERROR 2024-05-13T19:18:39.230068 1982,\n', 'ERROR 2024-05-13T19:18:39.230068 2460,\n', 'ERROR 2024-05-13T19:18:39.230068 2627,\n', 'ERROR 2024-05-13T19:18:39.230068 3246,\n', 'ERROR 2024-05-13T19:18:39.230068 3253,\n', 'ERROR 2024-05-13T19:18:39.230068 3268,\n', 'ERROR 2024-05-13T19:18:39.230068 3536,\n', 'ERROR 2024-05-13T19:18:39.230068 3846,\n', 'ERROR 2024-05-13T19:18:39.230068 3961,\n', 'ERROR 2024-05-13T19:18:39.230068 4183,\n', 'ERROR 2024-05-13T19:18:39.230068 4667,\n', 'ERROR 2024-05-13T19:18:39.230068 6585,\n', 'ERROR 2024-05-13T19:18:39.230068 6647,\n', 'ERROR 2024-05-13T19:18:39.230068 7273,\n', 'ERROR 2024-05-13T19:18:39.230068 9061,\n', 'ERROR 2024-05-13T19:18:39.230068 9383,\n', 'ERROR 2024-05-13T19:18:39.230068 10428,\n', 'ERROR 2024-05-13T19:18:39.230068 10929,\n', 'ERROR 2024-05-13T19:18:39.230068 11938,\n', 'ERROR 2024-05-13T19:18:39.230068 12033,\n', 'ERROR 2024-05-13T19:18:39.230068 12331,\n', 'ERROR 2024-05-13T19:18:39.230068 12562,\n', 'ERROR 2024-05-13T19:18:39.230068 13793,\n', 'ERROR 2024-05-13T19:18:39.230068 14157,\n', 'ERROR 2024-05-13T19:18:39.230068 14635,\n', 'ERROR 2024-05-13T19:18:39.230068 15265,\n', 'ERROR 2024-05-13T19:18:39.230068 15618,\n', 'ERROR 2024-05-13T19:18:39.230068 16553,\n', 'ERROR 2024-05-13T19:18:39.230068 16604,\n', 'ERROR 2024-05-13T19:18:39.230068 18362,\n', 'ERROR 2024-05-13T19:18:39.230068 18956,\n', 'ERROR 2024-05-13T19:18:39.230068 20075,\n', 'ERROR 2024-05-13T19:18:39.230068 21675,\n', 'ERROR 2024-05-13T19:18:39.230068 22520,\n', 'ERROR 2024-05-13T19:18:39.230068 26130,\n', 'ERROR 2024-05-13T19:18:39.230068 26161,\n', 'ERROR 2024-05-13T19:18:39.230068 26435,\n', 'ERROR 2024-05-13T19:18:39.230068 28279,\n', 'ERROR 2024-05-13T19:18:39.230068 29464,\n', 'ERROR 2024-05-13T19:18:39.230068 31650,\n', 'ERROR 2024-05-13T19:18:39.230068 32302,\n', 'ERROR 2024-05-13T19:18:39.230068 32470,\n', 'ERROR 2024-05-13T19:18:39.230068 36865,\n', 'ERROR 2024-05-13T19:18:39.230068 42863,\n', 'ERROR 2024-05-13T19:18:39.230068 47425,\n', 'ERROR 2024-05-13T19:18:39.230068 49870,\n', 'ERROR 2024-05-13T19:18:39.230068 50254,\n', 'ERROR 2024-05-13T19:18:39.230068 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 50360,\n', 'ERROR 2024-05-13T19:18:39.230068 50361,\n', 'ERROR 2024-05-13T19:18:39.230068 50362\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "torch_dtype": "float32",\n', 'ERROR 2024-05-13T19:18:39.230068 "transformers_version": "4.41.0.dev0",\n', 'ERROR 2024-05-13T19:18:39.230068 "use_cache": true,\n', 'ERROR 2024-05-13T19:18:39.230068 "use_weighted_layer_sum": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "vocab_size": 51865\n', 'ERROR 2024-05-13T19:18:39.230068 }\n', 'ERROR 2024-05-13T19:18:39.230068 loading configuration file config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/config.json\n', 'ERROR 2024-05-13T19:18:39.230068 Model config WhisperConfig {\n', 'ERROR 2024-05-13T19:18:39.230068 "_name_or_path": "distil-whisper/distil-large-v2",\n', 'ERROR 2024-05-13T19:18:39.230068 "activation_dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "activation_function": "gelu",\n', 'ERROR 2024-05-13T19:18:39.230068 "apply_spec_augment": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "architectures": [\n', 'ERROR 2024-05-13T19:18:39.230068 "WhisperForConditionalGeneration"\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "attention_dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "begin_suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 220,\n', 'ERROR 2024-05-13T19:18:39.230068 50257\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "bos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "classifier_proj_size": 256,\n', 'ERROR 2024-05-13T19:18:39.230068 "d_model": 1280,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_attention_heads": 20,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_ffn_dim": 5120,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_layerdrop": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_layers": 2,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_start_token_id": 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 "dropout": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_attention_heads": 20,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_ffn_dim": 5120,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_layerdrop": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "encoder_layers": 32,\n', 'ERROR 2024-05-13T19:18:39.230068 "eos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "forced_decoder_ids": [\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 50259\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 50359\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 3,\n', 'ERROR 2024-05-13T19:18:39.230068 50363\n', 'ERROR 2024-05-13T19:18:39.230068 ]\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "init_std": 0.02,\n', 'ERROR 2024-05-13T19:18:39.230068 "is_encoder_decoder": true,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_length": 10,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_min_masks": 0,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_feature_prob": 0.0,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_length": 10,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_min_masks": 2,\n', 'ERROR 2024-05-13T19:18:39.230068 "mask_time_prob": 0.05,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_length": 448,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_source_positions": 1500,\n', 'ERROR 2024-05-13T19:18:39.230068 "max_target_positions": 448,\n', 'ERROR 2024-05-13T19:18:39.230068 "median_filter_width": 7,\n', 'ERROR 2024-05-13T19:18:39.230068 "model_type": "whisper",\n', 'ERROR 2024-05-13T19:18:39.230068 "num_hidden_layers": 32,\n', 'ERROR 2024-05-13T19:18:39.230068 "num_mel_bins": 80,\n', 'ERROR 2024-05-13T19:18:39.230068 "pad_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "scale_embedding": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 7,\n', 'ERROR 2024-05-13T19:18:39.230068 8,\n', 'ERROR 2024-05-13T19:18:39.230068 9,\n', 'ERROR 2024-05-13T19:18:39.230068 10,\n', 'ERROR 2024-05-13T19:18:39.230068 14,\n', 'ERROR 2024-05-13T19:18:39.230068 25,\n', 'ERROR 2024-05-13T19:18:39.230068 26,\n', 'ERROR 2024-05-13T19:18:39.230068 27,\n', 'ERROR 2024-05-13T19:18:39.230068 28,\n', 'ERROR 2024-05-13T19:18:39.230068 29,\n', 'ERROR 2024-05-13T19:18:39.230068 31,\n', 'ERROR 2024-05-13T19:18:39.230068 58,\n', 'ERROR 2024-05-13T19:18:39.230068 59,\n', 'ERROR 2024-05-13T19:18:39.230068 60,\n', 'ERROR 2024-05-13T19:18:39.230068 61,\n', 'ERROR 2024-05-13T19:18:39.230068 62,\n', 'ERROR 2024-05-13T19:18:39.230068 63,\n', 'ERROR 2024-05-13T19:18:39.230068 90,\n', 'ERROR 2024-05-13T19:18:39.230068 91,\n', 'ERROR 2024-05-13T19:18:39.230068 92,\n', 'ERROR 2024-05-13T19:18:39.230068 93,\n', 'ERROR 2024-05-13T19:18:39.230068 359,\n', 'ERROR 2024-05-13T19:18:39.230068 503,\n', 'ERROR 2024-05-13T19:18:39.230068 522,\n', 'ERROR 2024-05-13T19:18:39.230068 542,\n', 'ERROR 2024-05-13T19:18:39.230068 873,\n', 'ERROR 2024-05-13T19:18:39.230068 893,\n', 'ERROR 2024-05-13T19:18:39.230068 902,\n', 'ERROR 2024-05-13T19:18:39.230068 918,\n', 'ERROR 2024-05-13T19:18:39.230068 922,\n', 'ERROR 2024-05-13T19:18:39.230068 931,\n', 'ERROR 2024-05-13T19:18:39.230068 1350,\n', 'ERROR 2024-05-13T19:18:39.230068 1853,\n', 'ERROR 2024-05-13T19:18:39.230068 1982,\n', 'ERROR 2024-05-13T19:18:39.230068 2460,\n', 'ERROR 2024-05-13T19:18:39.230068 2627,\n', 'ERROR 2024-05-13T19:18:39.230068 3246,\n', 'ERROR 2024-05-13T19:18:39.230068 3253,\n', 'ERROR 2024-05-13T19:18:39.230068 3268,\n', 'ERROR 2024-05-13T19:18:39.230068 3536,\n', 'ERROR 2024-05-13T19:18:39.230068 3846,\n', 'ERROR 2024-05-13T19:18:39.230068 3961,\n', 'ERROR 2024-05-13T19:18:39.230068 4183,\n', 'ERROR 2024-05-13T19:18:39.230068 4667,\n', 'ERROR 2024-05-13T19:18:39.230068 6585,\n', 'ERROR 2024-05-13T19:18:39.230068 6647,\n', 'ERROR 2024-05-13T19:18:39.230068 7273,\n', 'ERROR 2024-05-13T19:18:39.230068 9061,\n', 'ERROR 2024-05-13T19:18:39.230068 9383,\n', 'ERROR 2024-05-13T19:18:39.230068 10428,\n', 'ERROR 2024-05-13T19:18:39.230068 10929,\n', 'ERROR 2024-05-13T19:18:39.230068 11938,\n', 'ERROR 2024-05-13T19:18:39.230068 12033,\n', 'ERROR 2024-05-13T19:18:39.230068 12331,\n', 'ERROR 2024-05-13T19:18:39.230068 12562,\n', 'ERROR 2024-05-13T19:18:39.230068 13793,\n', 'ERROR 2024-05-13T19:18:39.230068 14157,\n', 'ERROR 2024-05-13T19:18:39.230068 14635,\n', 'ERROR 2024-05-13T19:18:39.230068 15265,\n', 'ERROR 2024-05-13T19:18:39.230068 15618,\n', 'ERROR 2024-05-13T19:18:39.230068 16553,\n', 'ERROR 2024-05-13T19:18:39.230068 16604,\n', 'ERROR 2024-05-13T19:18:39.230068 18362,\n', 'ERROR 2024-05-13T19:18:39.230068 18956,\n', 'ERROR 2024-05-13T19:18:39.230068 20075,\n', 'ERROR 2024-05-13T19:18:39.230068 21675,\n', 'ERROR 2024-05-13T19:18:39.230068 22520,\n', 'ERROR 2024-05-13T19:18:39.230068 26130,\n', 'ERROR 2024-05-13T19:18:39.230068 26161,\n', 'ERROR 2024-05-13T19:18:39.230068 26435,\n', 'ERROR 2024-05-13T19:18:39.230068 28279,\n', 'ERROR 2024-05-13T19:18:39.230068 29464,\n', 'ERROR 2024-05-13T19:18:39.230068 31650,\n', 'ERROR 2024-05-13T19:18:39.230068 32302,\n', 'ERROR 2024-05-13T19:18:39.230068 32470,\n', 'ERROR 2024-05-13T19:18:39.230068 36865,\n', 'ERROR 2024-05-13T19:18:39.230068 42863,\n', 'ERROR 2024-05-13T19:18:39.230068 47425,\n', 'ERROR 2024-05-13T19:18:39.230068 49870,\n', 'ERROR 2024-05-13T19:18:39.230068 50254,\n', 'ERROR 2024-05-13T19:18:39.230068 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 50360,\n', 'ERROR 2024-05-13T19:18:39.230068 50361,\n', 'ERROR 2024-05-13T19:18:39.230068 50362\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "torch_dtype": "float32",\n', 'ERROR 2024-05-13T19:18:39.230068 "transformers_version": "4.41.0.dev0",\n', 'ERROR 2024-05-13T19:18:39.230068 "use_cache": true,\n', 'ERROR 2024-05-13T19:18:39.230068 "use_weighted_layer_sum": false,\n', 'ERROR 2024-05-13T19:18:39.230068 "vocab_size": 51865\n', 'ERROR 2024-05-13T19:18:39.230068 }\n', 'ERROR 2024-05-13T19:18:39.230068 loading weights file model.safetensors from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/model.safetensors\n', 'ERROR 2024-05-13T19:18:39.230068 Generate config GenerationConfig {\n', 'ERROR 2024-05-13T19:18:39.230068 "begin_suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 220,\n', 'ERROR 2024-05-13T19:18:39.230068 50257\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "bos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "decoder_start_token_id": 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 "eos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "forced_decoder_ids": [\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 50259\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 50359\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 [\n', 'ERROR 2024-05-13T19:18:39.230068 3,\n', 'ERROR 2024-05-13T19:18:39.230068 50363\n', 'ERROR 2024-05-13T19:18:39.230068 ]\n', 'ERROR 2024-05-13T19:18:39.230068 ],\n', 'ERROR 2024-05-13T19:18:39.230068 "max_length": 448,\n', 'ERROR 2024-05-13T19:18:39.230068 "pad_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:39.230068 "suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:39.230068 1,\n', 'ERROR 2024-05-13T19:18:39.230068 2,\n', 'ERROR 2024-05-13T19:18:39.230068 7,\n', 'ERROR 2024-05-13T19:18:39.230068 8,\n', 'ERROR 2024-05-13T19:18:39.230068 9,\n', 'ERROR 2024-05-13T19:18:39.230068 10,\n', 'ERROR 2024-05-13T19:18:39.230068 14,\n', 'ERROR 2024-05-13T19:18:39.230068 25,\n', 'ERROR 2024-05-13T19:18:39.230068 26,\n', 'ERROR 2024-05-13T19:18:39.230068 27,\n', 'ERROR 2024-05-13T19:18:39.230068 28,\n', 'ERROR 2024-05-13T19:18:39.230068 29,\n', 'ERROR 2024-05-13T19:18:39.230068 31,\n', 'ERROR 2024-05-13T19:18:39.230068 58,\n', 'ERROR 2024-05-13T19:18:39.230068 59,\n', 'ERROR 2024-05-13T19:18:39.230068 60,\n', 'ERROR 2024-05-13T19:18:39.230068 61,\n', 'ERROR 2024-05-13T19:18:39.230068 62,\n', 'ERROR 2024-05-13T19:18:39.230068 63,\n', 'ERROR 2024-05-13T19:18:39.230068 90,\n', 'ERROR 2024-05-13T19:18:39.230068 91,\n', 'ERROR 2024-05-13T19:18:39.230068 92,\n', 'ERROR 2024-05-13T19:18:39.230068 93,\n', 'ERROR 2024-05-13T19:18:39.230068 359,\n', 'ERROR 2024-05-13T19:18:39.230068 503,\n', 'ERROR 2024-05-13T19:18:39.230068 522,\n', 'ERROR 2024-05-13T19:18:39.230068 542,\n', 'ERROR 2024-05-13T19:18:39.230068 873,\n', 'ERROR 2024-05-13T19:18:39.230068 893,\n', 'ERROR 2024-05-13T19:18:39.230068 902,\n', 'ERROR 2024-05-13T19:18:39.230068 918,\n', 'ERROR 2024-05-13T19:18:39.230068 922,\n', 'ERROR 2024-05-13T19:18:39.230068 931,\n', 'ERROR 2024-05-13T19:18:39.230068 1350,\n', 'ERROR 2024-05-13T19:18:39.230068 1853,\n', 'ERROR 2024-05-13T19:18:39.230068 1982,\n', 'ERROR 2024-05-13T19:18:39.230068 2460,\n', 'ERROR 2024-05-13T19:18:39.230068 2627,\n', 'ERROR 2024-05-13T19:18:39.230068 3246,\n', 'ERROR 2024-05-13T19:18:39.230068 3253,\n', 'ERROR 2024-05-13T19:18:39.230068 3268,\n', 'ERROR 2024-05-13T19:18:39.230068 3536,\n', 'ERROR 2024-05-13T19:18:39.230068 3846,\n', 'ERROR 2024-05-13T19:18:39.230068 3961,\n', 'ERROR 2024-05-13T19:18:39.230068 4183,\n', 'ERROR 2024-05-13T19:18:39.230068 4667,\n', 'ERROR 2024-05-13T19:18:39.230068 6585,\n', 'ERROR 2024-05-13T19:18:39.230068 6647,\n', 'ERROR 2024-05-13T19:18:39.230068 7273,\n', 'ERROR 2024-05-13T19:18:39.230068 9061,\n', 'ERROR 2024-05-13T19:18:39.230068 9383,\n', 'ERROR 2024-05-13T19:18:39.230068 10428,\n', 'ERROR 2024-05-13T19:18:39.230068 10929,\n', 'ERROR 2024-05-13T19:18:39.230068 11938,\n', 'ERROR 2024-05-13T19:18:39.230068 12033,\n', 'ERROR 2024-05-13T19:18:39.230068 12331,\n', 'ERROR 2024-05-13T19:18:39.230068 12562,\n', 'ERROR 2024-05-13T19:18:39.230068 13793,\n', 'ERROR 2024-05-13T19:18:39.230068 14157,\n', 'ERROR 2024-05-13T19:18:39.230068 14635,\n', 'ERROR 2024-05-13T19:18:39.230068 15265,\n', 'ERROR 2024-05-13T19:18:39.230068 15618,\n', 'ERROR 2024-05-13T19:18:39.230068 16553,\n', 'ERROR 2024-05-13T19:18:39.230068 16604,\n', 'ERROR 2024-05-13T19:18:39.230068 18362,\n', 'ERROR 2024-05-13T19:18:39.230068 18956,\n', 'ERROR 2024-05-13T19:18:39.230068 20075,\n', 'ERROR 2024-05-13T19:18:39.230068 21675,\n', 'ERROR 2024-05-13T19:18:39.230068 22520,\n', 'ERROR 2024-05-13T19:18:39.230068 26130,\n', 'ERROR 2024-05-13T19:18:39.230068 26161,\n', 'ERROR 2024-05-13T19:18:39.230068 26435,\n', 'ERROR 2024-05-13T19:18:39.230068 28279,\n', 'ERROR 2024-05-13T19:18:39.230068 29464,\n', 'ERROR 2024-05-13T19:18:39.230068 31650,\n', 'ERROR 2024-05-13T19:18:39.230068 32302,\n', 'ERROR 2024-05-13T19:18:39.230068 32470,\n', 'ERROR 2024-05-13T19:18:39.230068 36865,\n', 'ERROR 2024-05-13T19:18:39.230068 42863,\n', 'ERROR 2024-05-13T19:18:39.230068 47425,\n', 'ERROR 2024-05-13T19:18:39.230068 49870,\n', 'ERROR 2024-05-13T19:18:39.230068 50254,\n', 'ERROR 2024-05-13T19:18:39.230068 50258,\n', 'ERROR 2024-05-13T19:18:39.230068 50360,\n', 'ERROR 2024-05-13T19:18:39.230068 50361,\n', 'ERROR 2024-05-13T19:18:39.230068 50362\n', 'ERROR 2024-05-13T19:18:39.230068 ]\n', 'ERROR 2024-05-13T19:18:39.230068 }\n', 'ERROR 2024-05-13T19:18:41.238986 All model checkpoint weights were used when initializing WhisperForConditionalGeneration.\n', 'ERROR 2024-05-13T19:18:41.238986 All the weights of WhisperForConditionalGeneration were initialized from the model checkpoint at distil-whisper/distil-large-v2.\n', 'ERROR 2024-05-13T19:18:41.238986 If your task is similar to the task the model of the checkpoint was trained on, you can already use WhisperForConditionalGeneration for predictions without further training.\n', 'ERROR 2024-05-13T19:18:41.238986 loading configuration file generation_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/generation_config.json\n', 'ERROR 2024-05-13T19:18:41.238986 Generate config GenerationConfig {\n', 'ERROR 2024-05-13T19:18:41.238986 "begin_suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:41.238986 220,\n', 'ERROR 2024-05-13T19:18:41.238986 50257\n', 'ERROR 2024-05-13T19:18:41.238986 ],\n', 'ERROR 2024-05-13T19:18:41.238986 "bos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:41.238986 "decoder_start_token_id": 50258,\n', 'ERROR 2024-05-13T19:18:41.238986 "eos_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:41.238986 "forced_decoder_ids": [\n', 'ERROR 2024-05-13T19:18:41.238986 [\n', 'ERROR 2024-05-13T19:18:41.238986 1,\n', 'ERROR 2024-05-13T19:18:41.238986 50259\n', 'ERROR 2024-05-13T19:18:41.238986 ],\n', 'ERROR 2024-05-13T19:18:41.238986 [\n', 'ERROR 2024-05-13T19:18:41.238986 2,\n', 'ERROR 2024-05-13T19:18:41.238986 50359\n', 'ERROR 2024-05-13T19:18:41.238986 ],\n', 'ERROR 2024-05-13T19:18:41.238986 [\n', 'ERROR 2024-05-13T19:18:41.238986 3,\n', 'ERROR 2024-05-13T19:18:41.238986 50363\n', 'ERROR 2024-05-13T19:18:41.238986 ]\n', 'ERROR 2024-05-13T19:18:41.238986 ],\n', 'ERROR 2024-05-13T19:18:41.238986 "is_multilingual": true,\n', 'ERROR 2024-05-13T19:18:41.238986 "lang_to_id": {\n', 'ERROR 2024-05-13T19:18:41.238986 "<|af|>": 50327,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|am|>": 50334,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ar|>": 50272,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|as|>": 50350,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|az|>": 50304,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ba|>": 50355,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|be|>": 50330,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|bg|>": 50292,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|bn|>": 50302,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|bo|>": 50347,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|br|>": 50309,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|bs|>": 50315,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ca|>": 50270,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|cs|>": 50283,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|cy|>": 50297,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|da|>": 50285,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|de|>": 50261,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|el|>": 50281,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|en|>": 50259,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|es|>": 50262,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|et|>": 50307,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|eu|>": 50310,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|fa|>": 50300,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|fi|>": 50277,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|fo|>": 50338,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|fr|>": 50265,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|gl|>": 50319,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|gu|>": 50333,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|haw|>": 50352,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ha|>": 50354,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|he|>": 50279,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|hi|>": 50276,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|hr|>": 50291,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ht|>": 50339,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|hu|>": 50286,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|hy|>": 50312,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|id|>": 50275,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|is|>": 50311,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|it|>": 50274,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ja|>": 50266,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|jw|>": 50356,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ka|>": 50329,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|kk|>": 50316,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|km|>": 50323,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|kn|>": 50306,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ko|>": 50264,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|la|>": 50294,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|lb|>": 50345,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ln|>": 50353,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|lo|>": 50336,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|lt|>": 50293,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|lv|>": 50301,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mg|>": 50349,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mi|>": 50295,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mk|>": 50308,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ml|>": 50296,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mn|>": 50314,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mr|>": 50320,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ms|>": 50282,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|mt|>": 50343,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|my|>": 50346,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ne|>": 50313,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|nl|>": 50271,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|nn|>": 50342,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|no|>": 50288,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|oc|>": 50328,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|pa|>": 50321,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|pl|>": 50269,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ps|>": 50340,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|pt|>": 50267,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ro|>": 50284,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ru|>": 50263,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sa|>": 50344,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sd|>": 50332,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|si|>": 50322,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sk|>": 50298,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sl|>": 50305,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sn|>": 50324,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|so|>": 50326,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sq|>": 50317,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sr|>": 50303,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|su|>": 50357,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sv|>": 50273,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|sw|>": 50318,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ta|>": 50287,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|te|>": 50299,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|tg|>": 50331,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|th|>": 50289,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|tk|>": 50341,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|tl|>": 50348,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|tr|>": 50268,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|tt|>": 50351,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|uk|>": 50280,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|ur|>": 50290,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|uz|>": 50337,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|vi|>": 50278,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|yi|>": 50335,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|yo|>": 50325,\n', 'ERROR 2024-05-13T19:18:41.238986 "<|zh|>": 50260\n', 'ERROR 2024-05-13T19:18:41.238986 },\n', 'ERROR 2024-05-13T19:18:41.238986 "language": "<|en|>",\n', 'ERROR 2024-05-13T19:18:41.238986 "max_initial_timestamp_index": 50,\n', 'ERROR 2024-05-13T19:18:41.238986 "max_length": 448,\n', 'ERROR 2024-05-13T19:18:41.238986 "no_timestamps_token_id": 50363,\n', 'ERROR 2024-05-13T19:18:41.238986 "pad_token_id": 50257,\n', 'ERROR 2024-05-13T19:18:41.238986 "prev_sot_token_id": 50361,\n', 'ERROR 2024-05-13T19:18:41.238986 "return_timestamps": false,\n', 'ERROR 2024-05-13T19:18:41.238986 "suppress_tokens": [\n', 'ERROR 2024-05-13T19:18:41.238986 1,\n', 'ERROR 2024-05-13T19:18:41.238986 2,\n', 'ERROR 2024-05-13T19:18:41.238986 7,\n', 'ERROR 2024-05-13T19:18:41.238986 8,\n', 'ERROR 2024-05-13T19:18:41.238986 9,\n', 'ERROR 2024-05-13T19:18:41.238986 10,\n', 'ERROR 2024-05-13T19:18:41.238986 14,\n', 'ERROR 2024-05-13T19:18:41.238986 25,\n', 'ERROR 2024-05-13T19:18:41.238986 26,\n', 'ERROR 2024-05-13T19:18:41.238986 27,\n', 'ERROR 2024-05-13T19:18:41.238986 28,\n', 'ERROR 2024-05-13T19:18:41.238986 29,\n', 'ERROR 2024-05-13T19:18:41.238986 31,\n', 'ERROR 2024-05-13T19:18:41.238986 58,\n', 'ERROR 2024-05-13T19:18:41.238986 59,\n', 'ERROR 2024-05-13T19:18:41.238986 60,\n', 'ERROR 2024-05-13T19:18:41.238986 61,\n', 'ERROR 2024-05-13T19:18:41.238986 62,\n', 'ERROR 2024-05-13T19:18:41.238986 63,\n', 'ERROR 2024-05-13T19:18:41.238986 90,\n', 'ERROR 2024-05-13T19:18:41.238986 91,\n', 'ERROR 2024-05-13T19:18:41.238986 92,\n', 'ERROR 2024-05-13T19:18:41.238986 93,\n', 'ERROR 2024-05-13T19:18:41.238986 359,\n', 'ERROR 2024-05-13T19:18:41.238986 503,\n', 'ERROR 2024-05-13T19:18:41.238986 522,\n', 'ERROR 2024-05-13T19:18:41.238986 542,\n', 'ERROR 2024-05-13T19:18:41.238986 873,\n', 'ERROR 2024-05-13T19:18:41.238986 893,\n', 'ERROR 2024-05-13T19:18:41.238986 902,\n', 'ERROR 2024-05-13T19:18:41.238986 918,\n', 'ERROR 2024-05-13T19:18:41.238986 922,\n', 'ERROR 2024-05-13T19:18:41.238986 931,\n', 'ERROR 2024-05-13T19:18:41.238986 1350,\n', 'ERROR 2024-05-13T19:18:41.238986 1853,\n', 'ERROR 2024-05-13T19:18:41.238986 1982,\n', 'ERROR 2024-05-13T19:18:41.238986 2460,\n', 'ERROR 2024-05-13T19:18:41.238986 2627,\n', 'ERROR 2024-05-13T19:18:41.238986 3246,\n', 'ERROR 2024-05-13T19:18:41.238986 3253,\n', 'ERROR 2024-05-13T19:18:41.238986 3268,\n', 'ERROR 2024-05-13T19:18:41.238986 3536,\n', 'ERROR 2024-05-13T19:18:41.238986 3846,\n', 'ERROR 2024-05-13T19:18:41.238986 3961,\n', 'ERROR 2024-05-13T19:18:41.238986 4183,\n', 'ERROR 2024-05-13T19:18:41.238986 4667,\n', 'ERROR 2024-05-13T19:18:41.238986 6585,\n', 'ERROR 2024-05-13T19:18:41.238986 6647,\n', 'ERROR 2024-05-13T19:18:41.238986 7273,\n', 'ERROR 2024-05-13T19:18:41.238986 9061,\n', 'ERROR 2024-05-13T19:18:41.238986 9383,\n', 'ERROR 2024-05-13T19:18:41.238986 10428,\n', 'ERROR 2024-05-13T19:18:41.238986 10929,\n', 'ERROR 2024-05-13T19:18:41.238986 11938,\n', 'ERROR 2024-05-13T19:18:41.238986 12033,\n', 'ERROR 2024-05-13T19:18:41.238986 12331,\n', 'ERROR 2024-05-13T19:18:41.238986 12562,\n', 'ERROR 2024-05-13T19:18:41.238986 13793,\n', 'ERROR 2024-05-13T19:18:41.238986 14157,\n', 'ERROR 2024-05-13T19:18:41.238986 14635,\n', 'ERROR 2024-05-13T19:18:41.238986 15265,\n', 'ERROR 2024-05-13T19:18:41.238986 15618,\n', 'ERROR 2024-05-13T19:18:41.238986 16553,\n', 'ERROR 2024-05-13T19:18:41.238986 16604,\n', 'ERROR 2024-05-13T19:18:41.238986 18362,\n', 'ERROR 2024-05-13T19:18:41.238986 18956,\n', 'ERROR 2024-05-13T19:18:41.238986 20075,\n', 'ERROR 2024-05-13T19:18:41.238986 21675,\n', 'ERROR 2024-05-13T19:18:41.238986 22520,\n', 'ERROR 2024-05-13T19:18:41.238986 26130,\n', 'ERROR 2024-05-13T19:18:41.238986 26161,\n', 'ERROR 2024-05-13T19:18:41.238986 26435,\n', 'ERROR 2024-05-13T19:18:41.238986 28279,\n', 'ERROR 2024-05-13T19:18:41.238986 29464,\n', 'ERROR 2024-05-13T19:18:41.238986 31650,\n', 'ERROR 2024-05-13T19:18:41.238986 32302,\n', 'ERROR 2024-05-13T19:18:41.238986 32470,\n', 'ERROR 2024-05-13T19:18:41.238986 36865,\n', 'ERROR 2024-05-13T19:18:41.238986 42863,\n', 'ERROR 2024-05-13T19:18:41.238986 47425,\n', 'ERROR 2024-05-13T19:18:41.238986 49870,\n', 'ERROR 2024-05-13T19:18:41.238986 50254,\n', 'ERROR 2024-05-13T19:18:41.238986 50258,\n', 'ERROR 2024-05-13T19:18:41.238986 50360,\n', 'ERROR 2024-05-13T19:18:41.238986 50361,\n', 'ERROR 2024-05-13T19:18:41.238986 50362\n', 'ERROR 2024-05-13T19:18:41.238986 ],\n', 'ERROR 2024-05-13T19:18:41.238986 "task": "transcribe",\n', 'ERROR 2024-05-13T19:18:41.238986 "task_to_id": {\n', 'ERROR 2024-05-13T19:18:41.238986 "transcribe": 50359,\n', 'ERROR 2024-05-13T19:18:41.238986 "translate": 50358\n', 'ERROR 2024-05-13T19:18:41.238986 },\n', 'ERROR 2024-05-13T19:18:41.238986 "use_scan": false\n', 'ERROR 2024-05-13T19:18:41.238986 }\n', 'ERROR 2024-05-13T19:18:41.238986 loading file vocab.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/vocab.json\n', 'ERROR 2024-05-13T19:18:41.238986 loading file tokenizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer.json\n', 'ERROR 2024-05-13T19:18:41.238986 loading file merges.txt from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/merges.txt\n', 'ERROR 2024-05-13T19:18:41.238986 loading file normalizer.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/normalizer.json\n', 'ERROR 2024-05-13T19:18:41.238986 loading file added_tokens.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/added_tokens.json\n', 'ERROR 2024-05-13T19:18:41.238986 loading file special_tokens_map.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/special_tokens_map.json\n', 'ERROR 2024-05-13T19:18:41.238986 loading file tokenizer_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/tokenizer_config.json\n', 'ERROR 2024-05-13T19:18:41.238986 Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n', 'ERROR 2024-05-13T19:18:41.238986 loading configuration file preprocessor_config.json from cache at /raid/.cache/huggingface/models--distil-whisper--distil-large-v2/snapshots/66bb165856c86b9eae9dba7830c0cd7d859f4ef4/preprocessor_config.json\n', 'ERROR 2024-05-13T19:18:41.238986 Feature extractor WhisperFeatureExtractor {\n', 'ERROR 2024-05-13T19:18:41.238986 "chunk_length": 30,\n', 'ERROR 2024-05-13T19:18:41.238986 "feature_extractor_type": "WhisperFeatureExtractor",\n', 'ERROR 2024-05-13T19:18:41.238986 "feature_size": 80,\n', 'ERROR 2024-05-13T19:18:41.238986 "hop_length": 160,\n', 'ERROR 2024-05-13T19:18:41.238986 "n_fft": 400,\n', 'ERROR 2024-05-13T19:18:41.238986 "n_samples": 480000,\n', 'ERROR 2024-05-13T19:18:41.238986 "nb_max_frames": 3000,\n', 'ERROR 2024-05-13T19:18:41.238986 "padding_side": "right",\n', 'ERROR 2024-05-13T19:18:41.238986 "padding_value": 0.0,\n', 'ERROR 2024-05-13T19:18:41.238986 "processor_class": "WhisperProcessor",\n', 'ERROR 2024-05-13T19:18:41.238986 "return_attention_mask": false,\n', 'ERROR 2024-05-13T19:18:41.238986 "sampling_rate": 16000\n', 'ERROR 2024-05-13T19:18:41.238986 }\n', "ERROR 2024-05-13T19:18:41.238986 Disabling tokenizer parallelism, we're using DataLoader multithreading already\n", 'ERROR 2024-05-13T19:18:43.240189 You have passed task=transcribe, but also have set `forced_decoder_ids` to [[1, 50259], [2, 50359], [3, 50363]] which creates a conflict. `forced_decoder_ids` will be ignored in favor of task=transcribe.\n', 'ERROR 2024-05-13T19:18:47.242157 Train steps ... : 13%|███████▌ | 128/1016 [14:37<4:00:55, 16.28s/it]\n', '2024-05-13T19:18:46.651466 Eval results for step (127 / 1016 | Eval Loss: 4.316526412963867 | Eval clap: 0.4042370617389679 | Eval wer: 28.030303030303028 |)\n']}}, 'dropped': 0}} +2024-05-13 21:19:50,392 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:50,427 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:52,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:19:52,116 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:19:52,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:52,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:54,697 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:54,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:55,034 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:19:55,276 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:19:56,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:56,871 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:58,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:19:58,991 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:00,703 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:01,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:01,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:01,278 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:03,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:03,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:05,280 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:05,857 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:06,847 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:20:07,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:20:07,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:20:07,281 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:07,283 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:07,319 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:09,088 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:20:09,089 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:20:09,089 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:20:09,089 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:20:09,282 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:20:09,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:09,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:11,090 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:11,283 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:11,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:11,482 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:13,614 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:13,660 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:15,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:15,808 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:16,091 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:17,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:17,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:19,639 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:20:20,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:20,046 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:21,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:21,289 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:22,071 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:22,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:22,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:20:22,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:20:24,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:24,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:26,271 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:26,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:26,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:28,506 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:28,535 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:30,607 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:30,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:31,294 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:31,752 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:32,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:32,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:34,755 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:34,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:36,811 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:36,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:36,889 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:37,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:20:37,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:20:37,297 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:40,910 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:20:40,932 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:40,943 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:41,299 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:42,189 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:42,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:42,982 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:43,431 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:20:43,432 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:20:43,432 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:20:43,433 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:20:44,301 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:20:45,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:45,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:45,301 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:47,126 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:47,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:47,302 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:47,800 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:49,215 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:49,265 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:49,642 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:20:51,364 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:51,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:52,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:20:52,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:20:53,270 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:53,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:53,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:55,306 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:20:55,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:55,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:57,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:57,809 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:58,296 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:20:59,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:20:59,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:02,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:02,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:03,298 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:04,168 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:04,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:06,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:06,275 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:07,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:21:07,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:21:07,312 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:08,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:08,325 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:08,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:10,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:10,404 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:14,163 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:14,549 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:21:14,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:14,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:15,316 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:16,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:16,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:18,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:18,697 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:18,923 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:21:18,923 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:21:18,924 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:21:18,924 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:21:19,318 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:19,318 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:21:19,645 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:21:19,646 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:21,319 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:22,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:22,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:22,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:21:22,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:21:23,320 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:24,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:24,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:25,321 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:25,598 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:26,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:26,298 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:27,322 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:28,365 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:28,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:30,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:30,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:31,453 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:32,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:32,680 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:34,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:34,810 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:36,454 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:36,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:36,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:37,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:21:37,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:21:38,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:39,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:41,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:41,099 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:42,341 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:43,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:43,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:45,331 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:45,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:45,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:47,564 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:48,106 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:21:48,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:48,148 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:49,647 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:21:50,170 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:50,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:52,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:21:52,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:21:52,214 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:52,226 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:52,254 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:21:52,323 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:21:52,323 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:21:52,324 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:21:52,334 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:21:53,325 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:53,335 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:21:54,248 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:54,271 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:56,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:56,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:58,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:58,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:21:58,389 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:21:59,337 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:00,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:00,386 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:02,443 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:02,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:03,339 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:04,030 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:04,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:04,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:05,340 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:06,593 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:06,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:07,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:22:07,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:22:07,342 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:08,638 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:08,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:09,966 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:10,681 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:10,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:12,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:12,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:14,750 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:14,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:14,967 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:16,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:16,800 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:19,649 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:22:20,650 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:20,912 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:22:20,965 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:20,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:22,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:22:22,117 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:22:23,019 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:23,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:23,828 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:22:23,831 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:22:23,831 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:22:23,831 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:22:24,351 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:22:25,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:25,102 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:25,351 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:25,832 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:27,175 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:27,209 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:29,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:29,317 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:30,834 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:31,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:31,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:33,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:33,571 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:35,356 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:35,677 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:35,711 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:36,030 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:37,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:22:37,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:22:37,746 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:37,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:39,358 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:39,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:39,938 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:41,730 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:42,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:42,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:43,360 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:44,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:44,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:46,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:46,291 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:46,796 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:47,362 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:48,054 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:22:48,055 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:22:48,055 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:22:48,056 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:22:48,363 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:22:48,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:48,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:49,363 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:22:49,652 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:22:50,524 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:50,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:52,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:22:52,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:22:52,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:54,562 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:22:54,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:54,671 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:56,729 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:56,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:57,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:22:58,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:22:58,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:00,839 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:00,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:01,370 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:02,880 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:02,905 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:03,249 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:04,935 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:04,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:06,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:06,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:07,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:23:07,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:23:08,288 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:09,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:09,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:11,375 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:12,388 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:12,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:13,488 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:14,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:14,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:16,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:16,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:17,378 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:18,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:18,516 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:18,877 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:19,654 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:23:20,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:20,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:22,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:23:22,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:23:22,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:22,589 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:23,381 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:24,250 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:24,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:23:24,681 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:23:24,681 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:23:24,682 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:23:25,382 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:25,382 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:23:25,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:25,839 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:27,383 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:29,384 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:29,666 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:29,684 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:23:29,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:29,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:31,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:31,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:33,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:33,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:34,667 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:35,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:35,897 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:37,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:23:37,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:23:37,926 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:37,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:39,388 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:39,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:39,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:39,985 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:42,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:42,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:44,066 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:44,089 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:44,986 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:46,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:46,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:48,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:48,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:49,656 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:23:50,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:50,305 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:50,658 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:51,393 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:23:52,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:23:52,118 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:23:52,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:52,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:54,418 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:54,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:56,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:23:56,458 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:56,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:58,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:58,512 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:23:59,397 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:01,976 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:02,382 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:24:02,442 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:02,463 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:02,949 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:24:02,950 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:24:02,950 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:24:02,951 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:24:03,399 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:03,399 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:24:05,400 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:05,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:05,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:07,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:24:07,119 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:24:07,248 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:07,401 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:07,750 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:07,764 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:09,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:09,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:11,402 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:11,819 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:11,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:13,058 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:15,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:15,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:17,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:17,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:18,059 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:19,346 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:19,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:19,659 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:24:21,396 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:21,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:22,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:24:22,119 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:24:23,258 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:23,435 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:23,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:25,482 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:25,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:28,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:28,819 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:28,832 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:30,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:30,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:31,412 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:32,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:32,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:34,038 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:34,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:34,945 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:37,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:24:37,119 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:24:38,254 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:24:38,256 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:24:38,256 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:24:38,256 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:24:38,416 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:24:38,910 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:24:38,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:38,948 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:39,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:39,417 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:40,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:40,984 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:43,008 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:43,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:44,577 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:45,051 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:45,419 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:46,276 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:48,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:48,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:49,421 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:49,661 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:24:49,662 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:50,366 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:50,386 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:51,422 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:52,119 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:24:52,119 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:24:52,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:52,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:53,423 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:24:54,458 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:55,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:24:55,862 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:58,019 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:24:58,037 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:00,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:00,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:00,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:02,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:02,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:04,433 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:04,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:05,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:06,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:06,599 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:07,119 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:25:07,119 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:25:07,670 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:25:07,671 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:25:07,671 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:25:07,672 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:25:08,432 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:25:08,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:08,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:09,433 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:10,672 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:10,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:10,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:14,729 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:25:15,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:15,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:15,673 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:17,269 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:17,292 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:17,437 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:19,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:19,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:19,664 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:25:21,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:21,589 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:21,666 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:22,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:25:22,120 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:25:23,440 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:23,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:23,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:25,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:25,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:27,270 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:27,442 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:28,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:28,066 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:30,153 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:30,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:31,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:31,522 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:25:31,523 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:25:31,523 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:25:31,524 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:25:32,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:32,271 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:32,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:25:32,524 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:33,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:34,327 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:34,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:36,508 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:36,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:37,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:25:37,120 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:25:38,283 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:38,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:38,702 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:40,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:40,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:42,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:42,952 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:43,284 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:45,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:45,032 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:47,455 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:48,685 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:48,743 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:25:49,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:49,234 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:49,667 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:25:51,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:51,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:52,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:25:52,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:25:53,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:53,479 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:54,334 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:25:55,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:55,568 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:57,461 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:25:57,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:57,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:59,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:25:59,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:00,033 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:01,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:01,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:03,465 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:04,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:04,047 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:05,725 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:06,161 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:06,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:07,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:26:07,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:26:08,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:08,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:09,468 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:10,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:10,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:10,895 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:10,953 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:26:10,954 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:26:10,955 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:26:10,955 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:26:11,469 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:11,469 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:26:12,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:12,613 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:13,470 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:14,670 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:14,696 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:15,472 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:16,415 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:16,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:16,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:18,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:18,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:19,669 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:26:21,670 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:22,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:26:22,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:26:22,857 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:26:22,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:22,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:24,917 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:24,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:25,477 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:26,976 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:26,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:27,638 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:29,045 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:29,075 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:31,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:31,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:32,639 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:33,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:33,343 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:35,457 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:35,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:37,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:26:37,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:26:37,484 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:37,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:37,681 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:38,340 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:39,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:39,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:41,926 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:41,949 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:43,487 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:43,751 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:44,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:44,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:46,100 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:46,134 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:47,626 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:26:47,627 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:26:47,627 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:26:47,627 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:26:48,211 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:48,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:48,489 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:26:49,490 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:49,630 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:49,672 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:26:50,312 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:50,330 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:51,491 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:52,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:26:52,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:26:52,353 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:52,370 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:55,461 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:26:55,493 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:56,357 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:26:56,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:56,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:57,494 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:26:58,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:26:58,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:00,462 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:00,642 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:00,665 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:02,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:02,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:04,872 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:04,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:05,463 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:06,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:07,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:07,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:27:07,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:27:09,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:09,144 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:11,298 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:11,300 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:11,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:13,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:13,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:15,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:15,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:16,299 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:17,660 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:17,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:19,504 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:19,674 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:27:19,774 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:19,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:21,676 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:21,910 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:21,927 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:22,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:27:22,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:27:24,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:24,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:26,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:26,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:26,719 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:26,815 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:27:26,816 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:27:26,816 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:27:26,816 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:27:27,508 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:27,508 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:27:29,509 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:30,235 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:27:30,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:30,316 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:31,817 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:32,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:32,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:34,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:34,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:35,515 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:36,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:36,466 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:36,881 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:37,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:27:37,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:27:37,517 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:38,507 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:38,522 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:40,653 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:40,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:41,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:42,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:42,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:42,876 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:43,524 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:27:44,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:44,812 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:46,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:46,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:47,877 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:49,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:49,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:49,678 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:27:51,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:51,157 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:52,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:27:52,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:27:53,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:53,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:53,866 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:55,275 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:55,287 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:57,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:57,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:58,866 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:27:59,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:27:59,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:01,562 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:28:01,563 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:28:01,563 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:28:01,564 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:28:02,537 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:28:03,538 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:04,565 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:04,688 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:28:04,709 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:04,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:06,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:06,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:07,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:28:07,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:28:08,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:08,806 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:10,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:10,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:10,845 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:11,541 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:12,875 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:12,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:14,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:14,937 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:16,011 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:16,965 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:16,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:17,544 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:19,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:19,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:19,679 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:28:21,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:21,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:21,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:21,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:22,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:28:22,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:28:24,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:24,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:25,547 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:26,219 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:26,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:26,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:26,517 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:28:26,518 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:28:26,518 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:28:26,518 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:28:26,548 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:28:27,548 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:28,456 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:28,470 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:29,549 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:30,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:30,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:31,519 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:32,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:32,547 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:34,576 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:34,590 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:36,520 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:37,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:28:37,121 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:28:38,605 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:28:38,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:38,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:40,673 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:40,684 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:41,559 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:42,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:42,729 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:43,556 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:44,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:44,764 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:46,561 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:47,068 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:48,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:49,682 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:28:50,282 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:50,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:51,683 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:52,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:28:52,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:28:52,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:52,329 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:53,561 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:28:54,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:54,365 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:56,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:56,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:56,936 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:28:58,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:28:58,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:00,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:00,507 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:01,565 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:02,522 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:02,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:02,545 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:04,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:04,588 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:05,567 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:06,618 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:06,638 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:07,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:29:07,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:29:07,554 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:07,612 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:29:07,613 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:29:07,613 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:29:07,613 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:29:08,568 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:29:08,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:08,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:09,569 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:11,570 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:12,789 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:29:12,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:12,829 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:13,021 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:14,881 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:14,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:16,926 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:16,943 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:18,022 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:18,970 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:18,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:19,685 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:29:21,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:21,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:21,575 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:22,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:29:22,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:29:23,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:23,081 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:23,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:25,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:25,176 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:27,206 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:27,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:28,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:29,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:29,257 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:31,292 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:31,305 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:33,404 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:33,434 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:33,581 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:33,653 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:35,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:35,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:37,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:29:37,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:29:37,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:37,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:39,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:39,833 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:39,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:41,585 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:41,984 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:42,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:44,263 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:44,264 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:29:44,265 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:29:44,266 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:29:44,266 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:29:44,587 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:29:45,587 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:46,216 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:29:46,293 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:46,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:48,409 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:48,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:49,589 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:49,688 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:49,689 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:29:50,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:50,537 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:51,590 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:52,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:29:52,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:29:52,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:52,660 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:53,591 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:29:54,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:54,730 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:29:54,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:56,824 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:56,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:58,935 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:58,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:29:59,731 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:01,028 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:01,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:03,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:03,158 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:04,732 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:05,233 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:05,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:07,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:30:07,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:30:07,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:07,492 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:09,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:09,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:10,270 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:11,599 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:11,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:11,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:13,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:13,964 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:16,016 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:16,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:16,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:18,909 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:30:18,910 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:30:18,911 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:30:18,911 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:30:19,602 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:19,603 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:30:19,690 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:30:19,842 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:30:20,312 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:20,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:21,603 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:21,692 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:22,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:30:22,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:30:22,416 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:22,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:24,563 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:24,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:25,605 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:26,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:26,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:27,252 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:28,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:28,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:29,606 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:30,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:30,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:32,921 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:32,937 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:33,218 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:33,608 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:35,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:35,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:35,610 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:37,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:30:37,122 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:30:37,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:37,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:38,313 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:39,208 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:39,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:41,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:41,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:43,314 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:43,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:43,464 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:45,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:45,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:47,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:47,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:48,315 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:49,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:30:49,171 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:30:49,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:30:49,172 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:30:49,617 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:49,617 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:30:49,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:49,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:49,692 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:30:51,618 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:30:52,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:30:52,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:30:53,486 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:30:53,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:54,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:54,287 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:30:56,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:56,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:58,233 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:58,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:30:59,622 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:00,096 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:00,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:00,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:02,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:02,425 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:04,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:04,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:05,624 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:05,814 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:06,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:06,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:07,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:31:07,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:31:08,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:08,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:09,626 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:10,765 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:10,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:11,627 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:11,796 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:12,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:12,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:13,094 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:31:13,096 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:31:13,096 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:31:13,097 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:31:13,628 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:13,629 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:31:14,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:15,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:15,630 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:17,098 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:17,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:17,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:19,242 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:19,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:19,694 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:31:21,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:21,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:22,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:31:22,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:31:22,332 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:23,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:23,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:27,213 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:31:27,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:27,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:27,636 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:27,952 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:30,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:30,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:32,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:32,253 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:32,953 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:34,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:34,290 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:36,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:36,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:37,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:31:37,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:31:38,157 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:38,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:38,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:39,641 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:40,394 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:40,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:42,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:42,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:43,158 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:44,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:44,547 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:45,644 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:46,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:46,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:48,597 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:48,855 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:48,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:49,696 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:31:50,982 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:51,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:51,647 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:52,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:31:52,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:31:52,736 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:31:52,739 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:31:52,739 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:31:52,739 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:31:53,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:53,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:53,648 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:53,648 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:31:53,740 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:31:55,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:55,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:55,649 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:31:57,269 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:57,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:31:59,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:01,416 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:32:01,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:01,495 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:03,517 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:03,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:04,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:05,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:05,568 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:07,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:32:07,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:32:07,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:07,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:07,656 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:09,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:09,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:09,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:11,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:11,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:13,890 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:13,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:14,347 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:15,997 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:16,034 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:18,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:18,187 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:19,662 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:19,699 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:32:19,700 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:20,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:20,276 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:22,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:32:22,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:32:22,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:22,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:24,458 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:24,496 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:25,271 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:26,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:26,696 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:27,665 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:28,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:28,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:30,729 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:31,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:31,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:32,204 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:32:32,205 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:32:32,205 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:32:32,205 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:32:32,668 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:32:33,669 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:35,325 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:32:35,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:35,382 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:35,935 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:37,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:32:37,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:32:37,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:37,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:37,671 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:39,463 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:39,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:39,672 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:41,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:41,522 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:41,542 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:43,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:43,617 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:43,674 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:32:45,741 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:45,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:46,620 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:47,823 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:47,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:49,700 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:32:49,945 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:49,972 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:51,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:52,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:52,049 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:52,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:32:52,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:32:54,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:54,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:56,204 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:56,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:57,319 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:32:58,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:32:58,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:00,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:00,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:02,320 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:02,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:02,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:04,443 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:04,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:05,683 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:07,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:33:07,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:33:08,317 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:08,488 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:33:08,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:08,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:11,782 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:11,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:13,688 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:13,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:13,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:14,274 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:15,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:15,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:17,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:17,991 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:19,275 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:19,703 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:33:20,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:20,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:22,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:22,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:22,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:33:22,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:33:24,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:24,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:24,292 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:26,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:26,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:28,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:28,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:29,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:30,224 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:30,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:33,613 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:33,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:34,294 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:35,661 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:35,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:37,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:33:37,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:33:37,698 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:37,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:37,739 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:39,699 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:39,758 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:39,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:40,126 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:41,700 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:42,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 21:33:42,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 21:33:42,193 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:33:42,241 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:33:42,242 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:33:42,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav +2024-05-13 21:33:42,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav +2024-05-13 21:33:42,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 21:33:42,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 21:33:43,175 INFO wandb-upload_1:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmpw31229o0 +2024-05-13 21:33:43,351 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/izo3er6j-media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json +2024-05-13 21:33:43,759 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep254 - {'id': 'QXJ0aWZhY3Q6ODM0Nzg0OTMy', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI1MzkxMg==', 'latestArtifact': None}} +2024-05-13 21:33:43,760 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:43,760 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json with policy now +2024-05-13 21:33:43,760 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:33:43,760 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:33:43,813 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/jd743gyg-media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav +2024-05-13 21:33:43,857 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/5u23cw49-media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav +2024-05-13 21:33:43,939 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/rp7nv9pt-media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav +2024-05-13 21:33:43,940 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/u1p9sxmq-media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav +2024-05-13 21:33:43,973 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/nqi7q8oq-media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav +2024-05-13 21:33:43,977 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/79cry1c2-media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav +2024-05-13 21:33:44,006 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/t9xp6l6n-media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav +2024-05-13 21:33:44,197 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8k8rzequ-media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json +2024-05-13 21:33:44,707 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:33:44,753 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:33:44,753 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,753 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav with policy now +2024-05-13 21:33:44,753 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,753 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav with policy now +2024-05-13 21:33:44,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,754 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav with policy now +2024-05-13 21:33:44,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,754 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav with policy now +2024-05-13 21:33:44,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,754 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav with policy now +2024-05-13 21:33:44,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,754 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav with policy now +2024-05-13 21:33:44,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:33:44,755 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav with policy now +2024-05-13 21:33:44,865 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:33:44,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:44,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:45,708 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:45,755 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:45,778 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ikvtwf18-media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav +2024-05-13 21:33:45,784 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/niixjtsd-media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav +2024-05-13 21:33:45,785 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/exh9xy90-media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav +2024-05-13 21:33:45,812 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/rwtt25mx-media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav +2024-05-13 21:33:45,826 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/koxx4in5-media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav +2024-05-13 21:33:45,831 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/nl2ym3yq-media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav +2024-05-13 21:33:45,840 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/0owfk6bw-media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav +2024-05-13 21:33:46,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:46,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:47,575 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:33:47,577 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:33:47,578 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:33:47,578 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:33:47,709 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:47,709 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:33:49,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:49,044 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:49,707 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:33:49,712 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:51,067 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:51,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:51,569 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:51,713 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:33:52,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:33:52,126 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:33:54,484 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:54,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:56,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:56,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:57,265 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:33:58,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:33:58,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:00,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:00,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:02,266 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:02,710 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:02,728 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:04,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:04,762 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:06,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:06,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:07,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:34:07,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:34:07,297 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:08,829 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:08,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:10,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:10,908 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:12,297 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:12,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:12,991 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:13,728 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:16,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:16,335 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:17,414 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:19,708 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:34:20,269 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:34:20,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:20,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:21,734 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:22,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:34:22,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:34:22,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:22,369 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:23,295 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:24,391 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:24,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:26,425 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:26,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:27,738 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:28,460 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:28,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:28,684 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:29,627 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:34:29,627 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:34:29,628 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:34:29,628 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:34:29,739 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:34:31,741 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:31,870 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:31,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:33,742 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:33,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:33,959 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:34,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:35,744 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:36,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:36,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:37,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:34:37,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:34:38,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:38,093 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:40,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:40,146 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:40,357 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:42,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:42,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:44,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:44,233 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:45,357 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:46,257 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:46,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:48,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:48,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:49,712 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:34:50,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:50,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:50,713 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:52,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:34:52,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:34:53,757 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:34:54,475 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:34:54,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:54,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:56,292 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:34:56,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:56,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:58,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:34:58,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:00,924 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:00,951 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:01,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:02,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:03,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:03,764 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:05,049 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:05,064 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:06,721 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:07,094 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:07,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:07,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:35:07,124 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:35:08,163 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:35:08,164 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:35:08,164 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:35:08,165 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:35:08,768 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:35:09,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:09,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:09,769 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:11,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:11,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:12,165 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:13,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:13,257 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:13,772 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:15,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:15,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:17,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:17,366 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:17,537 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:17,774 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:19,432 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:19,469 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:19,714 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:35:19,776 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:21,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:21,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:22,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:35:22,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:35:23,256 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:23,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:23,673 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:27,909 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:35:27,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:27,995 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:28,258 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:30,069 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:30,084 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:32,218 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:32,365 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:33,457 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:33,786 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:34,429 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:34,442 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:36,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:36,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:37,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:35:37,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:35:38,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:38,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:39,287 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:40,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:40,633 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:42,666 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:42,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:43,657 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:35:43,658 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:35:43,658 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:35:43,658 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:35:43,792 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:35:44,659 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:44,705 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:44,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:45,793 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:46,737 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:46,753 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:48,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:48,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:49,660 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:49,716 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:35:50,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:50,836 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:51,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:52,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:35:52,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:35:52,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:52,870 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:54,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:54,935 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:55,534 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:35:55,800 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:35:56,995 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:57,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:35:59,803 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:00,762 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:36:01,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:01,216 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:01,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:01,805 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:03,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:03,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:05,424 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:05,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:06,178 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:07,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:36:07,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:36:07,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:07,508 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:09,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:09,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:09,905 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:36:09,906 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:36:09,906 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:36:09,906 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:36:10,811 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:36:11,776 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:11,811 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:11,815 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:11,907 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:13,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:13,944 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:16,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:16,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:16,909 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:18,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:18,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:19,719 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:36:20,300 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:20,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:21,918 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:22,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:36:22,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:36:22,464 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:22,491 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:23,818 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:24,588 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:24,634 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:26,718 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:26,758 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:27,308 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:28,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:28,890 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:29,825 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:30,975 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:31,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:32,738 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:33,827 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:34,616 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:36:34,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:34,690 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:36,738 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:36,758 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:37,125 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:36:37,125 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:36:37,830 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:38,267 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:38,819 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:38,830 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:39,408 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:36:39,409 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:36:39,409 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:36:39,410 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:36:39,831 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:39,831 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:36:40,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:40,879 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:41,832 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:36:42,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:42,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:44,241 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:44,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:45,000 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:47,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:47,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:49,069 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:49,081 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:49,242 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:49,722 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:36:51,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:51,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:52,126 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:36:52,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:36:53,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:53,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:54,602 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:36:55,188 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:55,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:57,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:57,279 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:59,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:36:59,440 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:00,448 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:01,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:01,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:01,844 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:03,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:03,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:05,449 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:07,126 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:37:07,126 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:37:07,900 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:37:07,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:07,968 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:09,849 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:10,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:10,072 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:10,513 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:12,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:12,150 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:13,851 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:14,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:14,259 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:16,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:16,373 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:16,397 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:17,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:37:17,258 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:37:17,259 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:37:17,259 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:37:17,854 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:17,854 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:37:18,545 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:18,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:19,726 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:37:19,855 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:20,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:20,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:21,856 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:22,059 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:22,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:37:22,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:37:22,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:22,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:23,858 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:24,957 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:24,984 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:27,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:27,131 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:27,306 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:29,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:29,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:31,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:31,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:32,306 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:33,460 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:33,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:35,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:35,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:37,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:37:37,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:37:37,310 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:37,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:37,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:39,867 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:41,861 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:37:41,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:41,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:42,499 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:44,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:44,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:46,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:46,157 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:47,521 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:47,871 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:48,282 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:48,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:49,726 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:37:50,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:50,476 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:52,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:37:52,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:37:52,303 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:37:52,304 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:37:52,304 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:37:52,304 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:37:52,571 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:52,606 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:52,874 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:52,874 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:37:53,305 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:53,875 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:54,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:54,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:56,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:56,819 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:57,877 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:37:58,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:37:58,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:37:58,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:00,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:00,946 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:01,879 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:02,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:02,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:03,881 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:03,906 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:06,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:06,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:07,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:38:07,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:38:08,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:08,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:09,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:10,356 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:10,377 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:12,402 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:12,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:14,349 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:16,370 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:38:16,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:16,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:17,889 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:18,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:18,550 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:19,729 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:38:19,731 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:20,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:20,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:22,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:38:22,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:38:22,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:22,614 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:24,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:24,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:25,287 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:26,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:26,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:27,824 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:38:27,825 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:38:27,825 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:38:27,825 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:38:27,895 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:38:28,896 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:29,897 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:30,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:30,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:30,826 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:32,153 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:32,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:34,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:34,204 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:35,900 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:36,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:36,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:36,422 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:37,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:38:37,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:38:38,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:38,279 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:40,302 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:40,312 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:41,475 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:41,903 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:42,335 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:42,346 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:43,904 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:45,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:45,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:46,804 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:47,700 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:47,710 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:47,907 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:49,731 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:38:51,677 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:38:51,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:51,718 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:52,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:38:52,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:52,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:38:53,740 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:53,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:55,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:55,808 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:56,617 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:38:56,618 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:38:56,618 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:38:56,618 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:38:56,912 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:56,912 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:38:57,619 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:38:57,829 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:57,846 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:57,913 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:38:59,875 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:38:59,886 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:01,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:01,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:02,619 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:03,968 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:03,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:06,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:06,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:07,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:39:07,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:39:08,171 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:08,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:08,299 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:10,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:10,322 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:11,920 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:12,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:12,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:13,500 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:14,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:14,444 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:16,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:16,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:18,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:18,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:19,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:19,734 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:39:19,924 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:20,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:20,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:22,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:39:22,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:39:24,533 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:25,928 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:25,999 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:39:26,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:26,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:27,930 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:28,069 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:28,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:30,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:30,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:30,178 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:30,242 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:39:30,243 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:39:30,243 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:39:30,244 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:39:30,932 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:30,932 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:39:31,932 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:32,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:32,274 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:33,933 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:34,533 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:34,574 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:35,341 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:37,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:39:37,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:39:37,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:37,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:39,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:39,733 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:41,265 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:41,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:41,807 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:43,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:43,842 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:45,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:45,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:46,268 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:47,965 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:48,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:49,737 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:39:50,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:50,044 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:51,739 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:39:52,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:52,089 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:52,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:39:52,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:39:53,946 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:39:54,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:54,187 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:56,215 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:56,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:39:56,859 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:00,414 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:40:00,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:00,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:01,950 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:02,481 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:02,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:02,785 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:04,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:04,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:06,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:06,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:07,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:40:07,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:40:07,954 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:08,312 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:08,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:08,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:10,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:10,728 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:11,212 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:40:11,213 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:40:11,213 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:40:11,214 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:40:11,957 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:11,957 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:40:12,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:12,781 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:12,958 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:13,944 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:14,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:14,812 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:15,960 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:16,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:16,859 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:17,961 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:18,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:18,925 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:19,067 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:19,739 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:40:20,955 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:20,966 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:22,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:40:22,129 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:40:23,066 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:23,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:24,301 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:25,131 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:25,156 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:27,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:27,216 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:29,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:29,269 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:29,302 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:33,608 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:40:33,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:33,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:33,971 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:34,516 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:35,722 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:35,747 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:37,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:40:37,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:40:38,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:38,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:40,296 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:41,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:41,159 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:41,976 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:43,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:43,287 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:45,373 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:45,442 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:45,838 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:46,890 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:40:46,891 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:40:46,891 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:40:46,892 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:40:46,979 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:40:47,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:47,527 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:47,980 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:48,981 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:49,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:49,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:49,742 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:40:51,395 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:51,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:51,914 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:51,983 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:52,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:40:52,128 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:40:54,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:54,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:55,985 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:56,215 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:56,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:56,458 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:40:57,986 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:40:58,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:40:58,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:00,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:00,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:01,614 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:02,392 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:02,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:04,435 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:04,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:06,615 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:07,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:41:07,129 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:41:08,553 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:41:08,578 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:08,588 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:09,994 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:10,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:10,632 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:12,357 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:12,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:12,670 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:14,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:14,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:16,742 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:16,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:17,358 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:18,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:18,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:19,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:41:19,106 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:41:19,106 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:41:19,106 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:41:19,743 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:41:20,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:20,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:41:20,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:20,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:21,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:22,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:41:22,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:41:23,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:23,099 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:23,277 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:25,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:25,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:26,006 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:27,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:27,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:28,977 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:29,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:29,432 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:30,008 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:31,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:31,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:34,011 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:34,514 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:34,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:34,592 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:36,012 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:37,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:41:37,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:41:37,503 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:37,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:40,309 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:40,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:40,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:44,977 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:41:44,978 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:41:44,978 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:41:44,978 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:41:45,018 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:41:45,979 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:46,019 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:47,019 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:41:47,633 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:41:47,679 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:47,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:49,746 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:41:50,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:50,872 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:51,748 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:52,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:41:52,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:41:53,808 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:53,838 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:56,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:56,816 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:57,321 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:41:59,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:41:59,871 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:00,027 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:02,566 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:02,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:02,827 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:05,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:05,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:06,030 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:07,129 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:42:07,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:42:08,253 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:08,838 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:08,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:11,942 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:11,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:12,034 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:14,035 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:14,074 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:14,994 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:15,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:15,715 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:42:15,716 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:42:15,716 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:42:15,717 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:42:16,036 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:42:17,037 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:17,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:18,012 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:18,038 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:19,597 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:19,748 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:42:20,039 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:21,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:21,066 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:22,130 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:42:22,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:42:24,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:24,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:25,299 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:27,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:27,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:30,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:30,216 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:30,300 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:35,850 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:36,049 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:37,130 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:42:37,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:42:39,413 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:42:40,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:40,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:41,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:43,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:44,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:44,054 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:46,493 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:47,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:47,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:49,754 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:42:50,058 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:50,074 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:50,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:51,736 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:51,781 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:42:51,782 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:42:51,782 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:42:51,782 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:42:52,060 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:42:52,130 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:42:52,130 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:42:53,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:53,060 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:53,071 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:54,061 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:56,062 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:56,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:56,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:57,592 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:42:58,063 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:42:59,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:42:59,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:02,131 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:02,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:02,593 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:05,166 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:05,208 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:07,130 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:43:07,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:43:08,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:08,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:08,294 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:11,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:11,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:14,073 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:14,159 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:14,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:14,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:17,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:17,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:19,160 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:19,758 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:43:20,496 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:20,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:22,078 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:22,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:43:22,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:43:23,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:23,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:24,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:26,802 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:43:26,803 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:43:26,803 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:43:26,803 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:43:27,081 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:43:28,082 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:29,082 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:29,804 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:32,084 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:33,077 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:43:33,856 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:33,874 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:35,530 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:36,086 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:36,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:36,806 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:37,130 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:43:37,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:43:38,088 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:39,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:39,926 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:41,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:42,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:42,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:45,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:46,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:46,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:49,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:49,100 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:49,759 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:43:50,095 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:43:51,761 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:52,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:43:52,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:43:52,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:52,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:55,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:55,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:57,286 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:43:58,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:58,287 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:43:59,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:43:59,148 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:43:59,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:43:59,148 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:44:00,101 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:00,101 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:44:01,102 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:01,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:01,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:03,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:04,276 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:04,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:06,105 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:07,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:44:07,132 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:44:07,251 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:07,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:08,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:10,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:10,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:12,108 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:13,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:13,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:13,868 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:14,109 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:16,110 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:16,209 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:16,260 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:19,254 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:19,763 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:44:22,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:44:22,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:44:24,262 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:24,968 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:44:24,969 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:44:24,969 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:44:24,970 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:44:25,104 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:44:25,117 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:44:25,138 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:25,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:26,118 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:27,118 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:28,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:28,153 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:29,970 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:31,176 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:31,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:34,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:34,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:34,972 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:37,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:37,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:44:37,131 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:44:37,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:38,125 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:39,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:39,864 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:40,341 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:42,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:42,542 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:45,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:45,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:45,550 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:46,130 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:47,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:48,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:49,764 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:44:50,765 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:50,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:50,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:52,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:44:52,132 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:44:52,134 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:53,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:53,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:56,136 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:56,197 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:44:56,249 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:44:56,250 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:44:56,250 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:44:56,251 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:44:56,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:56,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:57,136 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:57,137 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:44:58,137 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:44:59,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:44:59,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:00,138 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:01,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:01,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:02,165 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:04,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:04,700 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:07,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:45:07,132 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:45:07,284 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:12,284 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:13,098 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:45:13,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:13,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:15,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:15,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:17,635 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:18,148 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:18,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:18,827 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:19,767 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:45:21,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:21,683 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:22,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:45:22,132 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:45:23,355 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:24,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:24,542 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:26,153 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:27,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:27,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:28,614 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:30,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:30,034 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:30,155 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:32,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:32,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:32,973 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:45:32,974 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:45:32,975 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:45:32,975 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:45:33,156 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:45:33,975 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:34,157 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:35,145 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:35,157 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:35,166 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:37,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:45:37,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:45:37,812 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:37,834 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:38,159 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:39,705 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:40,160 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:40,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:40,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:43,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:43,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:44,706 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:45,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:45,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:48,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:48,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:49,707 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:49,769 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:45:51,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:51,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:52,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:45:52,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:45:54,168 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:45:55,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:45:59,478 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:45:59,503 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:45:59,514 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:00,260 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:02,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:02,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:04,174 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:04,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:04,694 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:05,602 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:07,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:46:07,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:46:07,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:46:07,226 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:07,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:07,333 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:46:07,334 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:46:07,334 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:46:08,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:08,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:46:09,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:09,816 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:09,838 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:11,046 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:12,179 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:12,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:12,535 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:15,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:15,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:16,181 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:16,934 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:17,756 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:17,771 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:18,183 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:19,771 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:46:20,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:20,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:22,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:46:22,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:22,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:46:23,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:23,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:25,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:25,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:27,318 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:28,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:28,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:31,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:31,190 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:32,192 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:32,704 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:33,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:33,848 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:36,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:36,420 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:37,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:46:37,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:46:38,311 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:39,690 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:46:39,691 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:46:39,692 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:46:39,692 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:46:40,196 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:46:41,196 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:42,197 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:43,693 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:45,517 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:46:45,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:45,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:48,200 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:48,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:48,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:49,470 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:49,774 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:46:50,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:50,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:52,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:46:52,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:46:52,202 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:53,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:53,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:54,496 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:46:56,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:56,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:56,204 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:58,206 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:46:58,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:58,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:46:59,942 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:01,444 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:01,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:04,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:04,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:04,944 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:06,427 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:47:06,429 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:47:06,429 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:47:06,429 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:47:06,925 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:06,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:07,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:47:07,134 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:47:07,210 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:07,210 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:47:08,211 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:09,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:09,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:10,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:12,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:12,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:15,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:15,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:15,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:17,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:17,817 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:19,777 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:47:20,315 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:20,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:20,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:22,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:47:22,134 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:47:22,219 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:23,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:23,290 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:25,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:30,223 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:30,856 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:31,276 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:47:32,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:32,550 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:34,225 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:35,084 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:35,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:36,013 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:37,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:47:37,134 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:47:37,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:37,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:38,227 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:39,315 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:47:39,315 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:47:39,316 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:47:39,316 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:47:40,229 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:40,229 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:47:40,386 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:40,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:41,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:41,229 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:42,230 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:42,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:42,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:45,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:45,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:46,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:47,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:47,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:49,779 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:47:50,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:50,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:50,234 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:47:51,780 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:52,134 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:47:52,135 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:47:52,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:52,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:55,201 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:55,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:56,942 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:47:57,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:47:57,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:00,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:00,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:01,944 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:02,458 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:02,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:04,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:04,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:06,944 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:07,134 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:48:07,134 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:48:07,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:07,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:12,321 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:14,778 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:48:15,730 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:15,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:17,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:18,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:18,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:19,781 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:48:20,633 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:20,658 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:22,134 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:48:22,135 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:48:22,571 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:23,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:23,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:24,252 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:25,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:25,343 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:26,253 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:27,765 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:27,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:28,254 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:28,525 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:28,647 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 21:48:28,648 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 21:48:28,654 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:48:28,722 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:48:28,724 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:48:29,255 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_492597073098578f0605.wav +2024-05-13 21:48:29,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json +2024-05-13 21:48:29,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 21:48:29,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 21:48:29,939 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/d4nh4djq-media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json +2024-05-13 21:48:30,170 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:30,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:30,345 INFO wandb-upload_6:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmpwdcyog3o +2024-05-13 21:48:30,503 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/9vkesifq-media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav +2024-05-13 21:48:30,545 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/jp8mgrso-media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav +2024-05-13 21:48:30,564 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/mr2sxip5-media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav +2024-05-13 21:48:30,564 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/xqs2j3rb-media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav +2024-05-13 21:48:30,574 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/246g1564-media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav +2024-05-13 21:48:30,644 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ppnuqlwy-media/audio/Speech samples/eval_381_492597073098578f0605.wav +2024-05-13 21:48:30,660 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/he6ljg25-media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav +2024-05-13 21:48:31,008 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep381 - {'id': 'QXJ0aWZhY3Q6ODM0ODA1OTg0', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI2Mzg5OA==', 'latestArtifact': None}} +2024-05-13 21:48:31,008 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,009 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json with policy now +2024-05-13 21:48:31,009 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:48:31,009 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:48:31,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:48:31,010 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav with policy now +2024-05-13 21:48:31,010 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav with policy now +2024-05-13 21:48:31,010 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav with policy now +2024-05-13 21:48:31,010 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav with policy now +2024-05-13 21:48:31,011 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,011 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_492597073098578f0605.wav with policy now +2024-05-13 21:48:31,011 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,011 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav with policy now +2024-05-13 21:48:31,011 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 21:48:31,011 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav with policy now +2024-05-13 21:48:31,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:48:31,457 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/2f8ll71e-media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json +2024-05-13 21:48:32,025 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/t0g9u7ur-media/audio/Speech samples/eval_381_492597073098578f0605.wav +2024-05-13 21:48:32,135 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/4xjdyiu6-media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav +2024-05-13 21:48:32,141 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/l91pm89r-media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav +2024-05-13 21:48:32,147 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8kqbn4sy-media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav +2024-05-13 21:48:32,148 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/cztr9kak-media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav +2024-05-13 21:48:32,153 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8ugzxajm-media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav +2024-05-13 21:48:32,178 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/mbhusgri-media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav +2024-05-13 21:48:32,258 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:32,794 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:32,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:33,259 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:34,013 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:35,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:35,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:37,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:48:37,135 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:48:38,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:38,085 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:39,358 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:40,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:40,679 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:43,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:43,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:44,267 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:44,397 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:46,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:46,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:48,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:48,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:49,627 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:49,783 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:48:50,271 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:51,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:51,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:52,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:48:52,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:48:52,272 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:48:54,636 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:48:59,421 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:48:59,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:59,468 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:48:59,637 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:01,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:01,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:04,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:04,638 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:04,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:07,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:49:07,135 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:49:07,283 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:07,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:09,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:09,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:10,329 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:10,860 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:49:10,861 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:49:10,862 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:49:10,862 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:49:11,285 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:49:12,286 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:12,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:12,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:13,286 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:15,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:15,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:15,863 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:18,033 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:18,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:19,786 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:49:20,293 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:20,634 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:20,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:21,788 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:22,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:49:22,135 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:49:23,259 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:23,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:25,903 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:25,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:26,297 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:27,086 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:28,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:28,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:30,299 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:31,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:31,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:32,301 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:33,026 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:33,098 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:49:33,099 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:49:33,099 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:49:33,099 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:49:33,301 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:49:33,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:33,666 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:34,302 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:35,303 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:36,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:36,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:37,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:49:37,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:49:38,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:43,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:44,215 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:49:45,490 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:45,517 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:48,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:48,219 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:48,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:49,789 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:49:50,846 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:50,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:52,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:49:52,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:49:52,314 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:49:53,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:53,488 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:54,312 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:49:56,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:56,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:58,860 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:58,880 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:49:59,313 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:00,320 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:01,551 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:01,580 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:04,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:04,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:04,518 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:06,324 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:06,764 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:06,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:07,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:50:07,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:50:09,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:09,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:09,685 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:12,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:12,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:12,328 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:12,785 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:50:12,787 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:50:12,788 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:50:12,788 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:50:13,328 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:13,328 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:50:14,329 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:14,789 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:14,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:14,917 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:16,330 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:17,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:17,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:19,791 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:50:19,793 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:20,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:20,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:22,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:50:22,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:50:22,962 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:22,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:25,340 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:28,338 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:30,426 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:31,105 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:50:32,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:32,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:34,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:34,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:35,428 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:37,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:50:37,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:50:37,384 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:37,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:40,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:40,074 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:40,346 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:40,514 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:42,823 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:42,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:45,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:45,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:46,076 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:46,351 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:48,231 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:48,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:49,773 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:50:49,773 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:50:49,774 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:50:49,774 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:50:49,793 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:50:50,354 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:50:51,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:51,032 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:51,355 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:51,794 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:52,135 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:50:52,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:50:52,356 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:53,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:53,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:54,357 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:56,212 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:56,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:56,358 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:50:57,385 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:50:59,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:50:59,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:01,842 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:01,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:02,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:04,363 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:04,396 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:04,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:07,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:07,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:07,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:51:07,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:51:08,296 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:09,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:09,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:13,297 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:16,370 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:17,678 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:51:18,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:18,853 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:18,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:19,796 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:51:21,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:21,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:21,711 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:51:21,712 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:51:21,712 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:51:21,713 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:51:22,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:51:22,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:51:22,373 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:22,374 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:51:23,374 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:24,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:24,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:24,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:26,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:26,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:28,377 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:29,413 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:29,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:29,533 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:32,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:32,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:32,379 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:34,380 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:34,535 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:34,643 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:34,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:36,381 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:37,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:51:37,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:51:37,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:37,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:40,002 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:40,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:40,303 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:42,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:42,740 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:45,305 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:45,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:45,448 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:47,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:47,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:49,798 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:51:50,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:50,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:50,799 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:51:52,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:51:52,136 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:51:52,438 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:52,459 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:52,658 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:51:52,659 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:51:52,659 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:51:52,660 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:51:53,391 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:53,391 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:51:54,391 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:51:54,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:54,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:51:56,661 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:02,396 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:02,655 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:03,119 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:52:03,681 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:03,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:06,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:06,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:07,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:52:07,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:52:08,329 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:08,399 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:09,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:09,397 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:11,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:11,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:12,401 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:13,586 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:14,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:14,382 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:14,402 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:15,662 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:52:15,663 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:52:15,663 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:52:15,663 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:52:16,404 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:16,404 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:52:16,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:16,880 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:17,404 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:18,664 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:19,290 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:19,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:19,801 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:52:21,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:21,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:22,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:52:22,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:52:24,288 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:24,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:24,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:26,824 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:26,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:29,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:29,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:29,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:31,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:31,820 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:34,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:34,283 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:34,414 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:35,075 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:36,693 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:36,720 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:37,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:52:37,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:52:38,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:39,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:40,359 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:42,419 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:45,879 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:46,404 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:52:47,653 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:47,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:48,422 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:49,803 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:52:50,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:50,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:51,500 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:52,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:52:52,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:52:52,424 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:52,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:52,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:53,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:52:53,363 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:52:53,364 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:52:53,364 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:52:53,425 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:52:54,425 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:55,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:55,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:55,426 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:56,426 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:52:57,456 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:52:57,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:52:57,658 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:00,275 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:00,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:02,457 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:02,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:02,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:05,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:05,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:07,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:53:07,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:53:07,817 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:08,395 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:08,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:10,433 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:11,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:11,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:12,818 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:13,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:13,872 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:16,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:16,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:18,714 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:19,193 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:19,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:19,805 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:53:20,438 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:21,908 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:21,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:22,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:53:22,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:53:24,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:24,528 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:24,710 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:26,441 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:27,934 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:53:27,936 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:53:27,936 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:53:27,936 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:53:28,443 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:53:29,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:29,937 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:30,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:32,312 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:53:32,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:32,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:32,445 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:34,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:34,848 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:34,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:35,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:37,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:53:37,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:53:37,562 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:37,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:40,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:40,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:41,327 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:42,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:42,886 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:44,450 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:45,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:45,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:47,260 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:48,148 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:48,168 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:49,808 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:53:50,761 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:50,797 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:52,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:53:52,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:53:52,304 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:53,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:53,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:56,157 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:56,180 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:57,624 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:53:58,458 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:53:58,889 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:53:58,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:01,071 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:01,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:02,625 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:03,811 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:54:03,812 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:54:03,812 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:54:03,813 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:54:03,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:03,943 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:04,461 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:54:05,462 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:06,462 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:06,631 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:06,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:07,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:54:07,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:54:08,314 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:09,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:09,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:10,466 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:13,744 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:14,468 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:17,505 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:54:17,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:17,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:18,471 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:18,882 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:19,810 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:54:20,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:20,201 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:20,472 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:22,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:54:22,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:54:22,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:22,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:24,274 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:25,566 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:25,593 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:28,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:28,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:29,275 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:30,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:30,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:33,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:33,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:34,276 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:36,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:36,356 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:37,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:54:37,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:54:37,390 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:54:37,390 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:54:37,391 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:54:37,391 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:54:37,481 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:54:38,481 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:39,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:39,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:39,392 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:39,481 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:41,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:41,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:44,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:44,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:44,392 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:46,484 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:47,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:47,037 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:49,643 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:49,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:49,812 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:54:49,813 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:52,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:54:52,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:54:52,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:52,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:52,487 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:54,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:54,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:54:54,961 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:54:56,489 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:58,490 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:58,843 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:54:58,844 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:54:58,844 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:54:58,845 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:54:59,491 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:54:59,491 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:55:00,491 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:00,845 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:02,667 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:55:03,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:03,964 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:05,846 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:06,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:06,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:07,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:55:07,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:55:09,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:09,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:11,302 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:12,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:12,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:14,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:14,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:16,501 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:17,034 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:17,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:17,627 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:19,815 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:55:20,232 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:20,260 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:22,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:55:22,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:22,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:55:22,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:22,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:25,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:25,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:26,507 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:27,260 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:28,164 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:28,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:30,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:30,666 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:32,510 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:32,544 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:33,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:33,322 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:35,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:35,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:36,513 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:37,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:55:37,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:55:37,879 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:37,935 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:55:37,936 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:55:37,936 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:55:37,936 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:55:38,514 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:55:38,542 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:38,568 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:39,514 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:40,515 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:41,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:41,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:42,516 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:43,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:48,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:49,241 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:55:49,818 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:55:50,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:50,292 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:52,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:55:52,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:55:52,903 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:52,931 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:53,354 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:55:54,524 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:55:55,500 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:55,531 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:58,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:58,105 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:55:58,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:00,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:00,713 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:03,378 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:03,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:03,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:06,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:06,030 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:06,531 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:07,137 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:56:07,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:56:08,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:08,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:09,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:11,534 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:11,560 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:12,535 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:14,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:14,224 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:14,869 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:15,632 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:56:15,634 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:56:15,634 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:56:15,634 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:56:16,538 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:16,538 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:56:16,816 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:16,842 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:17,538 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:19,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:19,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:19,819 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:56:20,540 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:20,821 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:21,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:22,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:22,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:56:22,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:56:22,542 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:24,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:24,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:26,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:27,311 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:27,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:32,247 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:32,549 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:35,272 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:56:36,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:36,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:37,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:56:37,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:56:37,332 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:39,174 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:39,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:41,889 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:41,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:42,333 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:44,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:44,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:46,556 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:47,337 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:47,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:47,356 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:49,822 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:56:49,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:49,991 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:51,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:56:51,387 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:56:51,387 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:56:51,388 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:56:51,559 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:56:52,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:56:52,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:56:52,479 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:52,560 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:52,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:52,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:53,560 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:56:55,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:55,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:57,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:58,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:56:58,290 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:56:58,563 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:00,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:00,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:02,566 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:03,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:03,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:03,355 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:04,567 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:05,833 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:05,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:06,568 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:07,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:57:07,138 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:57:08,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:08,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:09,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:11,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:11,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:14,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:14,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:14,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:19,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:19,825 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:57:21,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:57:21,695 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:57:21,695 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:57:21,696 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:57:22,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:57:22,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:57:22,344 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:57:22,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:22,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:57:23,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:23,438 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:23,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:24,335 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:26,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:26,061 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:28,618 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:28,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:29,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:31,331 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:31,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:32,583 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:33,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:33,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:34,917 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:36,586 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:36,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:36,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:37,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:57:37,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:57:39,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:39,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:40,588 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:40,665 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:41,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:41,891 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:44,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:44,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:44,591 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:44,659 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:57:44,659 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:57:44,660 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:57:44,660 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:57:45,592 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:45,592 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:57:46,593 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:57:46,661 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:47,089 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:47,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:49,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:49,772 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:49,828 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:57:51,829 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:52,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:57:52,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:57:52,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:52,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:55,037 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:55,069 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:57,263 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:57:57,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:57:57,671 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:00,339 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:00,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:02,399 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:02,602 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:07,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:58:07,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:58:08,311 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:08,457 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:58:08,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:08,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:10,607 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:11,234 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:11,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:13,629 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:13,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:13,889 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:16,610 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:16,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:16,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:19,472 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:19,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:19,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:19,830 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:58:20,613 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:21,681 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:58:21,682 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:58:21,682 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:58:21,682 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:58:22,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:58:22,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:58:22,214 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:22,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:22,614 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:22,614 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:58:23,615 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:24,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:24,773 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:24,949 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:26,617 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:27,541 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:27,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:29,950 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:30,188 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:30,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:32,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:32,867 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:34,950 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:35,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:35,498 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:37,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:58:37,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:58:38,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:38,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:38,623 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:40,350 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:40,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:40,830 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:43,485 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:43,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:45,351 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:46,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:46,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:48,629 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:49,832 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:58:50,833 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:52,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:58:52,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:58:52,631 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:55,454 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:58:55,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:55,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:55,917 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:58:55,980 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:58:55,981 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:58:55,981 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:58:55,982 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:58:56,633 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:58:57,634 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:58:58,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:58,068 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:58:58,635 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:00,637 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:00,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:00,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:01,111 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:02,638 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:03,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:03,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:06,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:06,201 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:06,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:07,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:59:07,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:59:08,848 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:08,874 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:11,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:11,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:11,996 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:12,643 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:14,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:14,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:16,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:16,997 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:17,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:19,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:19,736 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:19,836 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:59:22,139 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:22,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:59:22,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:59:22,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:22,451 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:25,136 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:25,159 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:26,651 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:27,272 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:27,700 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:27,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:30,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:30,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:31,647 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:59:31,647 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 21:59:31,648 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:59:31,648 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:59:31,655 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 21:59:32,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:32,656 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:33,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:33,122 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:33,657 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:37,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:59:37,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:59:38,360 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:38,660 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:41,223 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:59:42,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:42,473 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:42,663 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:43,934 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:44,664 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:45,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:45,071 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:46,665 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 21:59:47,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:47,721 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:49,837 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 21:59:49,839 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:50,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:50,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:52,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:59:52,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:59:53,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:53,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:55,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:59:55,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:56,021 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:58,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:59:58,802 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:00,337 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:01,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:01,427 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:03,577 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:00:03,578 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:00:03,578 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:00:03,578 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:00:03,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:00:04,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:04,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:04,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:05,579 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:05,676 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:06,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:06,771 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:07,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:00:07,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:00:09,432 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:09,451 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:11,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:12,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:12,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:12,680 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:14,932 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:14,946 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:16,433 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:17,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:17,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:18,684 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:19,840 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:00:20,292 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:20,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:22,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:22,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:00:22,141 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:00:22,686 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:24,688 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:24,870 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:00:24,872 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:00:24,872 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:00:24,872 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:00:25,688 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:25,689 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:00:26,690 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:27,873 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:28,043 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:00:28,284 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:29,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:32,091 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:32,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:32,874 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:34,810 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:34,826 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:37,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:00:37,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:00:37,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:37,603 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:38,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:40,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:40,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:42,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:43,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:43,132 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:44,158 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:45,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:45,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:48,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:48,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:49,159 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:49,842 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:00:51,547 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:51,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:52,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:00:52,140 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:00:52,706 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:54,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:54,178 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:54,335 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:00:56,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:56,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:58,710 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:00:59,396 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:00:59,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:00,095 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:02,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:02,051 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:02,712 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:03,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:01:03,185 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:01:03,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:01:03,186 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:01:03,712 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:01:04,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:04,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:04,713 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:05,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:05,714 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:06,714 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:07,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:01:07,141 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:01:07,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:07,309 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:10,301 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:15,171 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:01:15,201 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:15,216 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:15,302 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:17,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:17,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:19,845 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:01:20,524 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:20,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:20,723 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:20,846 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:22,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:01:22,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:01:23,075 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:23,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:25,603 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:25,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:26,309 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:28,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:28,276 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:30,730 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:31,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:31,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:31,952 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:33,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:33,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:36,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:36,463 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:36,733 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:37,038 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:37,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:01:37,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:01:39,132 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:39,153 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:39,409 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:01:40,218 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:01:40,218 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:01:40,218 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:01:40,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:40,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:01:41,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:41,781 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:41,800 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:42,219 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:44,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:44,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:44,739 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:46,740 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:46,941 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:46,979 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:47,873 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:49,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:49,631 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:49,848 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:01:52,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:01:52,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:01:52,184 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:52,216 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:53,310 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:54,745 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:01:58,462 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:01:59,938 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:01:59,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:01:59,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:02,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:02,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:03,462 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:05,275 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:05,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:06,752 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:07,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:02:07,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:02:08,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:08,046 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:09,302 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:10,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:10,778 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:12,629 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:02:12,630 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:02:12,630 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:02:12,631 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:02:12,756 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:02:13,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:13,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:13,757 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:14,631 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:14,757 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:16,185 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:16,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:18,759 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:18,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:18,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:19,850 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:02:19,851 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:21,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:21,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:22,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:02:22,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:02:22,762 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:23,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:23,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:24,763 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:25,610 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:26,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:26,219 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:28,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:28,515 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:30,657 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:30,767 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:30,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:30,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:32,768 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:33,228 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:33,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:35,711 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:35,741 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:36,000 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:37,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:02:37,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:02:41,369 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:42,961 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:02:43,000 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:43,014 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:45,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:45,395 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:46,370 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:47,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:47,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:49,853 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:02:50,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:50,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:51,854 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:52,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:02:52,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:02:52,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:52,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:54,993 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:55,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:57,364 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:02:57,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:57,404 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:58,783 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:02:59,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:02:59,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:00,785 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:02,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:02,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:02,787 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:02,788 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:03,127 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 22:03:03,127 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 22:03:03,133 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:03:03,189 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:03:03,191 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:03:03,788 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav +2024-05-13 22:03:03,788 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json +2024-05-13 22:03:03,788 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav +2024-05-13 22:03:03,788 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 22:03:03,789 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 22:03:03,987 INFO wandb-upload_6:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmp2ug83zl8 +2024-05-13 22:03:04,228 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/skxedlc6-media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json +2024-05-13 22:03:04,719 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep508 - {'id': 'QXJ0aWZhY3Q6ODM0ODM0NzUy', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI4MTIzMA==', 'latestArtifact': None}} +2024-05-13 22:03:04,719 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,719 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json with policy now +2024-05-13 22:03:04,720 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:03:04,720 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:03:04,721 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:03:04,721 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,721 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav with policy now +2024-05-13 22:03:04,721 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,721 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav with policy now +2024-05-13 22:03:04,721 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,721 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav with policy now +2024-05-13 22:03:04,721 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,721 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav with policy now +2024-05-13 22:03:04,723 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,723 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav with policy now +2024-05-13 22:03:04,723 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,723 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav with policy now +2024-05-13 22:03:04,723 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:03:04,723 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav with policy now +2024-05-13 22:03:04,745 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:04,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:04,788 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:03:04,862 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/viy58bog-media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav +2024-05-13 22:03:05,025 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/31xq2da1-media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav +2024-05-13 22:03:05,041 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ffnsvkzb-media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav +2024-05-13 22:03:05,055 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/k5bltd4d-media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav +2024-05-13 22:03:05,065 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/d1n4r5hq-media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav +2024-05-13 22:03:05,071 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/1xe9kgz5-media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav +2024-05-13 22:03:05,081 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/r4b78mdn-media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav +2024-05-13 22:03:05,156 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8m07u8eb-media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json +2024-05-13 22:03:05,790 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:05,978 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/cw1x5ci0-media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav +2024-05-13 22:03:06,164 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8m00leil-media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav +2024-05-13 22:03:06,180 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/aib73xk5-media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav +2024-05-13 22:03:06,183 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/sn8slo2m-media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav +2024-05-13 22:03:06,223 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/5yii1mxx-media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav +2024-05-13 22:03:06,790 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:07,073 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/jfvmb17v-media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav +2024-05-13 22:03:07,122 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/44m4arab-media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav +2024-05-13 22:03:07,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:03:07,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:03:07,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:07,370 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:08,349 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:09,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:09,878 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:12,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:12,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:13,350 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:15,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:15,329 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:17,995 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:18,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:18,351 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:19,855 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:03:21,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:03:21,133 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:03:21,133 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:03:21,709 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:03:21,802 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:03:22,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:03:22,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:03:22,803 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:23,804 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:24,342 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:25,736 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:03:25,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:25,787 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:26,806 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:28,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:28,166 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:28,807 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:30,252 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:31,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:31,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:33,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:33,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:35,253 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:36,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:36,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:37,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:03:37,144 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:03:39,021 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:39,050 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:40,347 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:41,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:41,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:44,403 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:44,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:45,347 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:47,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:47,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:48,820 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:49,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:49,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:49,858 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:03:50,859 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:52,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:03:52,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:03:52,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:52,335 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:54,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:55,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:56,486 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:03:56,826 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:03:57,633 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:03:57,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:00,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:00,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:00,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:04:01,000 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:04:01,000 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:04:01,001 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:04:01,829 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:01,829 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:04:02,001 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:02,830 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:02,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:02,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:06,833 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:07,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:04:07,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:04:07,369 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:10,836 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:11,047 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:04:11,084 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:11,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:12,635 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:12,838 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:13,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:13,541 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:16,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:16,148 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:17,636 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:18,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:18,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:19,860 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:04:21,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:21,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:22,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:04:22,145 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:04:23,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:24,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:24,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:26,732 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:26,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:28,292 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:28,850 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:29,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:29,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:32,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:32,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:33,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:34,690 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:34,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:36,962 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:04:36,963 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:04:36,964 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:04:36,964 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:04:37,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:04:37,145 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:04:37,328 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:37,346 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:37,857 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:37,858 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:04:38,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:38,858 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:40,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:40,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:42,756 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:42,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:43,764 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:44,862 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:45,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:45,611 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:48,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:48,309 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:48,865 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:49,006 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:49,863 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:04:50,867 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:52,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:04:52,145 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:04:54,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:04:54,870 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:04:55,947 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:04:55,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:55,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:58,497 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:58,511 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:04:59,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:01,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:01,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:03,858 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:03,879 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:04,354 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:05,710 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:05:05,711 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:05:05,712 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:05:05,712 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:05:05,879 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:05:06,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:06,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:06,879 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:07,145 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:05:07,146 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:05:07,880 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:09,284 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:09,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:09,355 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:12,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:12,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:14,356 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:14,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:14,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:16,886 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:17,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:17,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:19,613 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:19,865 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:05:20,096 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:20,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:22,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:05:22,145 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:05:22,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:22,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:22,889 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:25,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:25,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:25,393 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:27,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:27,951 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:28,893 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:30,425 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:30,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:30,507 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:30,894 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:31,778 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:05:31,780 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:05:31,780 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:05:31,780 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:05:31,895 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:05:32,815 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:32,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:32,896 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:33,896 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:35,781 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:37,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:05:37,145 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:05:39,813 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:05:39,847 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:39,859 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:40,957 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:42,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:42,208 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:42,903 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:44,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:44,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:45,958 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:47,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:47,063 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:49,468 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:49,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:49,867 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:05:51,869 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:51,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:51,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:52,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:05:52,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:05:54,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:54,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:56,774 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:56,797 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:56,913 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:05:57,369 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:05:59,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:05:59,253 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:01,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:01,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:02,485 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:02,916 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:04,146 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:04,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:06,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:06,622 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:07,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:06:07,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:06:08,286 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:08,920 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:09,045 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:09,067 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:10,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:06:10,147 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:06:10,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:06:10,148 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:06:10,922 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:06:11,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:11,490 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:11,922 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:12,923 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:13,728 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:13,880 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:13,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:14,924 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:16,926 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:19,683 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:19,869 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:06:21,399 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:06:22,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:22,021 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:22,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:06:22,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:06:24,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:24,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:25,283 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:27,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:27,129 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:29,780 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:29,823 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:30,284 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:32,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:32,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:35,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:35,224 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:35,583 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:36,939 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:37,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:06:37,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:06:37,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:37,977 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:40,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:40,634 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:41,319 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:43,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:43,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:44,945 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:45,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:45,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:46,365 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:47,762 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:06:47,763 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:06:47,763 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:06:47,764 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:06:47,948 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:06:48,176 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:48,187 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:48,949 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:49,873 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:06:49,949 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:50,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:50,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:51,536 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:52,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:06:52,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:06:52,951 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:53,425 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:53,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:56,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:56,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:56,954 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:06:57,325 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:06:58,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:58,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:06:58,955 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:02,326 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:06,826 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:07:06,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:06,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:07,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:07:07,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:07:07,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:09,490 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:09,520 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:12,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:12,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:12,965 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:13,309 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:14,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:14,810 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:17,373 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:17,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:18,310 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:19,874 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:07:20,033 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:07:20,034 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:07:20,034 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:07:20,034 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:07:20,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:20,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:20,970 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:20,970 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:07:21,970 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:22,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:07:22,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:07:22,741 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:22,777 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:23,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:25,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:25,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:26,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:28,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:28,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:28,683 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:30,976 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:30,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:31,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:33,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:33,716 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:33,803 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:34,979 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:36,237 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:36,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:36,981 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:37,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:07:37,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:07:38,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:38,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:39,331 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:41,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:41,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:44,156 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:44,170 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:44,332 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:49,276 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:07:49,277 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:07:49,277 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:07:49,277 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:07:49,876 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:07:49,877 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:49,990 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:07:50,991 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:51,873 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:07:51,992 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:07:52,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:07:52,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:07:52,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:53,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:55,335 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:07:55,864 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:55,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:58,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:07:58,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:00,729 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:01,001 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:01,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:01,316 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:03,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:04,004 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:05,730 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:06,739 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:06,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:07,005 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:07,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:08:07,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:08:09,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:09,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:11,603 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:11,946 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:11,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:13,010 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:14,524 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:14,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:15,012 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:16,194 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:08:16,195 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:08:16,195 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:08:16,195 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:08:17,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:17,013 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:08:17,032 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:17,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:18,014 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:19,015 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:19,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:19,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:19,880 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:08:22,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:08:22,147 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:08:22,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:22,265 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:22,291 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:24,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:24,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:27,365 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:27,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:27,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:29,022 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:30,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:30,042 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:32,367 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:37,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:08:37,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:08:37,371 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:37,948 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:08:37,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:37,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:40,586 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:40,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:41,030 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:43,169 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:43,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:43,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:45,997 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:46,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:47,034 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:48,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:48,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:48,928 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:49,882 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:08:51,435 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:51,457 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:52,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:08:52,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:08:53,039 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:53,983 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:54,041 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:08:54,042 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:08:54,042 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:08:54,042 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:08:54,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:54,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:55,044 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:55,044 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:08:56,045 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:56,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:56,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:59,047 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:08:59,334 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:08:59,335 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:08:59,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:01,048 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:01,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:02,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:04,335 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:04,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:04,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:07,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:09:07,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:09:07,365 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:07,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:09,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:10,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:10,233 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:12,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:12,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:14,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:15,403 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:15,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:17,059 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:19,413 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:19,884 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:09:22,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:09:22,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:09:23,144 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:09:23,171 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:23,188 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:24,833 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:25,065 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:25,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:25,802 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:28,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:28,469 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:28,966 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:09:28,967 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:09:28,967 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:09:28,968 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:09:29,067 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:09:29,968 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:30,068 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:31,068 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:31,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:31,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:33,747 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:33,764 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:35,071 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:35,391 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:36,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:36,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:37,072 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:37,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:09:37,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:09:38,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:39,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:39,073 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:41,380 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:41,643 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:41,661 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:44,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:44,327 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:46,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:46,899 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:46,916 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:49,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:49,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:49,887 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:09:51,889 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:09:52,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:52,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:09:52,148 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:09:52,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:54,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:54,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:55,084 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:09:57,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:57,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:09:57,703 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:00,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:00,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:02,275 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:10:02,276 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:10:02,276 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:10:02,277 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:10:03,090 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:10:03,277 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:04,091 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:05,092 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:07,148 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:10:07,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:10:07,929 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:10:07,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:07,977 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:09,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:10,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:10,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:11,095 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:13,353 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:13,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:14,473 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:15,098 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:15,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:15,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:17,099 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:18,408 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:18,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:19,101 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:19,732 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:19,889 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:10:20,973 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:21,000 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:22,148 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:10:22,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:10:23,670 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:23,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:25,332 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:26,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:26,501 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:29,180 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:29,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:30,333 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:31,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:31,873 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:32,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:10:32,347 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:10:32,347 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:10:32,347 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:10:33,112 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:10:34,113 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:34,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:34,492 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:35,113 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:35,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:37,148 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:10:37,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:10:37,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:37,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:39,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:39,871 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:40,369 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:42,512 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:42,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:45,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:45,120 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:45,134 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:45,775 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:49,893 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:10:50,894 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:51,130 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:52,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:10:52,150 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:10:53,054 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:10:53,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:53,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:55,617 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:55,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:56,271 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:10:57,136 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:58,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:58,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:10:59,138 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:10:59,809 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:10:59,810 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:10:59,810 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:10:59,810 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:11:00,139 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:11:00,857 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:00,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:01,140 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:01,811 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:02,142 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:03,462 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:03,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:06,316 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:06,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:06,812 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:07,148 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:11:07,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:11:09,071 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:09,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:11,149 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:11,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:11,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:12,404 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:14,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:14,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:17,027 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:17,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:17,405 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:19,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:19,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:19,895 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:11:22,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:11:22,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:11:22,208 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:22,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:23,160 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:23,359 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:24,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:24,979 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:27,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:27,586 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:28,359 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:30,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:30,215 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:31,168 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:33,793 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:35,172 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:37,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:11:37,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:11:37,261 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:11:37,343 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:11:37,343 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:11:37,344 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:11:38,175 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:11:39,176 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:39,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:39,556 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:11:40,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:40,557 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:40,580 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:41,177 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:43,123 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:43,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:43,179 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:11:44,659 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:45,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:45,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:48,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:48,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:49,660 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:49,897 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:11:51,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:51,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:52,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:11:52,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:11:53,684 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:53,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:55,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:11:56,335 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:56,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:58,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:11:59,003 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:01,197 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:01,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:01,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:01,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:04,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:04,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:06,325 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:07,085 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:07,100 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:07,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:12:07,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:12:09,204 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:09,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:09,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:11,504 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:12,288 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:12,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:13,136 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:12:13,137 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:12:13,137 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:12:13,138 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:12:13,208 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:12:14,209 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:14,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:14,891 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:15,210 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:16,930 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:17,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:17,570 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:19,214 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:19,900 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:12:21,216 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:22,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:12:22,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:12:22,296 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:23,218 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:25,334 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:12:25,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:25,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:27,297 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:27,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:27,997 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:30,757 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:30,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:32,298 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:33,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:33,395 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:36,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:36,126 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:37,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:12:37,149 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:12:37,321 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:38,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:38,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:39,235 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:41,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:41,358 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:42,321 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:43,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:43,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:45,677 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:12:45,678 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:12:45,678 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:12:45,678 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:12:46,241 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:12:46,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:46,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:47,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:47,679 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:48,244 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:49,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:49,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:49,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:12:51,972 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:51,994 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:52,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:12:52,150 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:12:53,248 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:53,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:54,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:54,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:57,253 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:12:57,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:57,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:12:58,976 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:12:59,254 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:00,028 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:00,049 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:01,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:02,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:02,710 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:04,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:07,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:13:07,150 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:13:09,327 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:10,552 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:13:10,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:10,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:13,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:13,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:13,349 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:13:13,350 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:13:13,351 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:13:13,351 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:13:14,269 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:13:14,352 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:15,270 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:15,871 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:15,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:16,272 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:18,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:18,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:19,352 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:19,905 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:13:21,218 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:21,240 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:22,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:13:22,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:13:23,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:23,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:24,279 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:24,899 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:26,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:26,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:29,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:29,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:30,095 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:30,285 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:31,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:31,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:34,289 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:34,533 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:34,553 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:35,456 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:37,138 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:37,149 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:37,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:13:37,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:13:38,293 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:39,185 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:13:39,186 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:13:39,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:13:39,187 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:13:39,294 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:39,294 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:13:39,651 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:39,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:40,295 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:41,188 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:42,175 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:42,198 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:44,746 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:44,776 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:46,189 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:47,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:47,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:49,907 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:13:50,304 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:13:51,909 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:52,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:13:52,150 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:13:55,326 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:13:55,364 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:55,386 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:57,331 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:13:57,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:13:58,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:00,722 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:00,739 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:02,316 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:02,916 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:03,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:03,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:06,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:06,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:07,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:14:07,150 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:14:08,321 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:08,360 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:08,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:08,792 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:11,508 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:11,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:12,325 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:13,896 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:14,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:14,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:15,418 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:14:15,419 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:14:15,419 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:14:15,420 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:14:16,329 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:16,329 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:14:16,756 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:16,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:17,330 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:18,331 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:19,330 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:19,353 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:19,675 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:19,910 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:14:20,333 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:21,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:21,941 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:22,150 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:14:22,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:14:24,622 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:24,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:25,342 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:27,279 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:27,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:30,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:30,061 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:30,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:32,713 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:32,733 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:35,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:37,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:14:37,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:14:40,353 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:40,553 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:14:40,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:40,614 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:40,678 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:43,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:43,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:45,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:45,719 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:45,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:48,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:48,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:49,912 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:14:50,364 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:50,913 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:51,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:51,046 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:52,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:14:52,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:14:53,756 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:14:53,757 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:14:53,757 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:14:53,757 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:14:53,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:53,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:54,368 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:54,368 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:14:55,369 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:56,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:56,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:56,758 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:14:58,372 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:14:59,021 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:14:59,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:01,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:01,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:02,376 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:02,480 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:04,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:04,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:04,378 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:06,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:06,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:07,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:15:07,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:15:08,284 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:09,386 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:09,418 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:11,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:12,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:13,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:14,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:14,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:17,170 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:17,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:18,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:19,915 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:15:20,393 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:22,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:15:22,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:15:23,328 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:25,024 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:15:25,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:25,074 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:27,739 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:15:27,740 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:15:27,740 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:15:27,740 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:15:27,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:27,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:28,401 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:28,401 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:15:28,741 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:29,402 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:30,463 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:30,484 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:33,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:33,248 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:34,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:34,407 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:35,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:35,874 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:37,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:15:37,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:15:38,411 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:38,457 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:38,482 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:39,436 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:41,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:41,042 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:42,415 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:43,618 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:43,642 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:44,417 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:44,651 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:46,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:46,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:49,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:49,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:49,652 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:49,917 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:15:51,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:51,812 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:52,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:15:52,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:15:54,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:54,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:55,251 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:15:55,433 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:15:55,434 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:15:55,434 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:15:55,613 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:15:56,430 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:56,430 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:15:57,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:57,185 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:57,431 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:15:59,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:15:59,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:00,614 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:02,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:02,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:05,616 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:06,439 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:07,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:16:07,151 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:16:10,041 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:16:10,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:10,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:11,283 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:12,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:13,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:13,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:15,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:15,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:16,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:18,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:18,333 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:18,449 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:19,920 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:16:20,451 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:20,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:20,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:21,970 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:22,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:16:22,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:16:22,476 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:16:22,477 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:16:22,477 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:16:22,478 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:16:23,453 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:23,453 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:16:23,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:23,593 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:24,454 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:26,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:26,178 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:27,478 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:28,847 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:28,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:31,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:31,627 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:32,459 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:33,141 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:34,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:34,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:37,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:37,064 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:37,151 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:16:37,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:16:38,380 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:39,693 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:39,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:42,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:42,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:43,381 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:45,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:45,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:46,470 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:47,702 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:47,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:48,870 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:49,922 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:16:52,152 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:16:52,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:16:52,475 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:54,307 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:56,951 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:16:56,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:57,022 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:58,481 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:16:59,554 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:16:59,681 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:16:59,700 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:00,001 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:17:00,002 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:17:00,002 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:17:00,002 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:17:00,482 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:00,483 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:17:01,483 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:02,026 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:02,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:04,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:04,424 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:05,003 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:06,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:06,732 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:07,152 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:17:07,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:17:08,493 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:09,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:09,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:10,077 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:11,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:11,586 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:13,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:14,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:15,078 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:16,487 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:16,512 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:18,883 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:18,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:19,924 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:17:20,926 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:21,388 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:21,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:22,153 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:17:22,154 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:17:23,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:23,836 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:26,242 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:26,259 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:26,352 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:28,793 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:28,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:31,291 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:31,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:31,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:36,517 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:36,602 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:37,152 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:17:37,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:17:38,518 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:38,782 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:17:38,802 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:38,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:40,519 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:41,195 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:41,214 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:41,469 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 22:17:41,469 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 22:17:41,476 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:17:41,520 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json +2024-05-13 22:17:41,520 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 22:17:41,530 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:17:41,531 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:17:41,942 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/xavg6oni-media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json +2024-05-13 22:17:42,240 INFO wandb-upload_0:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmp96rb8gl7 +2024-05-13 22:17:42,521 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav +2024-05-13 22:17:42,521 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav +2024-05-13 22:17:42,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 22:17:42,875 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep635 - {'id': 'QXJ0aWZhY3Q6ODM0ODUxMzg4', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI4Njc5MA==', 'latestArtifact': None}} +2024-05-13 22:17:42,880 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,882 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:42,882 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json with policy now +2024-05-13 22:17:42,895 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,900 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav with policy now +2024-05-13 22:17:42,901 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,901 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav with policy now +2024-05-13 22:17:42,901 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav with policy now +2024-05-13 22:17:42,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav with policy now +2024-05-13 22:17:42,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav with policy now +2024-05-13 22:17:42,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,903 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav with policy now +2024-05-13 22:17:42,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:17:42,903 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav with policy now +2024-05-13 22:17:43,522 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:43,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:43,740 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:43,751 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/4ai8emmm-media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json +2024-05-13 22:17:43,853 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/woueedf5-media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav +2024-05-13 22:17:43,979 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/h2ochj19-media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav +2024-05-13 22:17:44,169 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/3gsxo8k2-media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav +2024-05-13 22:17:44,263 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/smdoboj8-media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav +2024-05-13 22:17:44,282 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/sxh3z35v-media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav +2024-05-13 22:17:44,303 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/9hmhf2xq-media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav +2024-05-13 22:17:44,314 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/8bfu8i90-media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav +2024-05-13 22:17:44,525 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:44,959 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/315gc28l-media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav +2024-05-13 22:17:45,381 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/vn2mvetc-media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav +2024-05-13 22:17:45,406 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/cxek704t-media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav +2024-05-13 22:17:45,416 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/3qpf0vlv-media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav +2024-05-13 22:17:46,060 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/cmgjfhdc-media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav +2024-05-13 22:17:46,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:46,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:46,441 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/h55dzwfh-media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav +2024-05-13 22:17:46,484 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/zse5ms4e-media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav +2024-05-13 22:17:46,527 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:48,229 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:48,528 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:17:48,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:49,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:49,927 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:17:51,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:51,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:52,152 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:17:52,152 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:17:53,303 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:54,290 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:54,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:56,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:57,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:58,304 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:17:59,757 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:17:59,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:02,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:02,542 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:02,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:03,834 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:05,278 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:05,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:06,544 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:07,021 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:18:07,023 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:18:07,023 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:18:07,023 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:18:07,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:18:07,260 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:18:07,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:07,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:18:07,873 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:07,899 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:08,546 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:09,417 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:10,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:10,830 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:13,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:13,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:14,418 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:16,094 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:16,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:19,418 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:19,930 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:18:22,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:18:22,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:18:23,098 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:18:24,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:24,166 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:25,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:26,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:26,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:28,562 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:29,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:29,622 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:31,277 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:32,426 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:32,442 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:35,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:35,105 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:36,278 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:37,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:18:37,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:18:37,781 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:37,803 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:38,570 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:40,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:40,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:41,360 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:42,573 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:43,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:43,055 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:45,641 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:45,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:46,577 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:47,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:48,246 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:48,266 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:48,979 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:18:48,980 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:18:48,980 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:18:48,981 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:18:49,579 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:49,579 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:18:49,932 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:18:50,580 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:50,937 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:50,955 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:52,169 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:52,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:18:52,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:18:52,581 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:18:53,481 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:53,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:56,118 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:56,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:57,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:18:58,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:18:58,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:01,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:01,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:02,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:07,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:19:07,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:19:07,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:08,594 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:09,572 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:19:09,598 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:09,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:12,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:12,267 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:12,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:14,879 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:14,905 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:16,600 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:17,476 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:17,516 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:17,539 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:19,936 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:19:19,979 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:19,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:22,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:19:22,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:19:22,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:22,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:22,604 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:23,340 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:24,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:24,589 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:25,138 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:19:25,139 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:19:25,139 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:19:25,139 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:19:25,607 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:25,607 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:19:26,608 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:27,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:27,028 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:28,670 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:29,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:29,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:30,612 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:31,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:31,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:32,614 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:33,908 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:34,364 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:34,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:36,821 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:36,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:37,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:19:37,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:19:39,293 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:39,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:39,361 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:41,796 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:41,820 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:44,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:44,294 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:44,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:46,626 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:49,729 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:49,937 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:19:51,560 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:19:52,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:19:52,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:19:52,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:52,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:55,012 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:19:55,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:55,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:56,635 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:19:57,901 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:19:57,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:00,013 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:00,382 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:00,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:01,344 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:20:01,345 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:20:01,345 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:20:01,347 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:20:01,639 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:20:02,640 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:02,884 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:02,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:03,641 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:05,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:05,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:05,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:06,643 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:07,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:20:07,170 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:20:07,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:07,598 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:09,948 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:09,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:10,646 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:11,108 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:12,435 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:12,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:12,647 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:15,119 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:15,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:16,509 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:17,876 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:17,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:19,939 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:20:20,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:20,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:22,170 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:22,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:20:22,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:20:22,653 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:23,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:23,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:25,990 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:26,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:27,312 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:28,693 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:28,711 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:32,313 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:34,510 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:20:34,511 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:20:34,512 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:20:34,512 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:20:34,661 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:20:35,662 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:36,260 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:20:36,663 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:37,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:20:37,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:20:37,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:37,358 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:37,374 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:40,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:40,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:42,635 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:42,646 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:42,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:42,667 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:45,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:45,351 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:47,771 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:48,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:48,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:48,670 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:49,942 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:20:50,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:50,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:52,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:20:52,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:20:52,673 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:53,218 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:53,535 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:53,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:54,674 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:55,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:20:55,286 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:20:55,286 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:20:55,286 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:20:55,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:55,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:20:56,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:56,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:56,675 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:20:58,287 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:20:58,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:20:58,782 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:01,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:01,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:03,288 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:03,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:03,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:06,520 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:06,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:07,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:21:07,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:21:08,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:09,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:09,321 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:12,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:12,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:14,017 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:14,687 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:14,832 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:14,856 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:19,019 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:19,945 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:21:22,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:21:22,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:21:22,693 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:23,081 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:21:24,107 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:24,118 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:24,398 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:26,696 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:26,731 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:26,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:29,419 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:29,432 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:30,245 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:30,698 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:31,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:32,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:33,167 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:21:33,169 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:21:33,169 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:21:33,169 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:21:33,700 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:33,700 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:21:34,514 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:34,537 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:34,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:36,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:36,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:37,017 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:37,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:37,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:21:37,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:21:39,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:39,730 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:41,364 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:42,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:42,422 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:45,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:45,090 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:46,364 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:47,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:47,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:49,947 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:21:50,517 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:50,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:51,809 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:52,171 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:21:52,171 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:21:52,713 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:21:53,171 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:53,187 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:55,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:55,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:57,343 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:21:58,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:21:58,492 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:01,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:01,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:02,718 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:02,924 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:07,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:22:07,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:22:08,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:08,724 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:09,839 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:22:10,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:10,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:10,570 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:22:10,571 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:22:10,572 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:22:10,572 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:22:10,725 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:22:11,726 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:12,727 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:12,877 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:12,890 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:13,426 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:14,728 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:15,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:15,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:16,730 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:18,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:18,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:18,592 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:19,950 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:22:20,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:20,824 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:22,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:22:22,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:22:23,479 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:23,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:24,297 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:26,188 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:26,222 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:28,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:28,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:28,878 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:29,556 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:31,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:31,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:34,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:34,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:34,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:36,739 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:36,760 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:37,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:22:37,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:22:38,742 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:39,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:39,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:40,314 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:41,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:42,012 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:43,145 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:22:43,146 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:22:43,146 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:22:43,146 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:22:43,746 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:43,746 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:22:44,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:44,710 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:44,747 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:46,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:47,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:47,417 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:48,749 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:49,952 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:22:51,821 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:52,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:22:52,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:22:52,752 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:54,753 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:22:55,126 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:22:55,146 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:55,158 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:57,249 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:22:57,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:22:57,860 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:00,480 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:00,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:02,250 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:03,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:03,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:04,762 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:05,846 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:05,863 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:07,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:23:07,172 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:23:07,389 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:08,506 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:08,526 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:11,244 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:11,271 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:12,390 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:13,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:13,952 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:15,409 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:23:15,411 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:23:15,411 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:23:15,412 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:23:15,769 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:23:16,612 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:16,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:16,770 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:17,412 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:17,771 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:19,397 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:19,438 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:19,955 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:23:22,168 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:22,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:23:22,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:23:22,186 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:22,775 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:23,330 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:24,814 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:24,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:27,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:27,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:28,779 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:29,089 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:30,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:30,088 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:32,663 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:32,680 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:32,782 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:34,784 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:34,961 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:35,009 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:23:35,011 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:23:35,011 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:23:35,011 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:23:35,784 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:35,785 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:23:36,786 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:37,172 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:23:37,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:23:40,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:40,406 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:23:40,438 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:40,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:43,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:43,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:45,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:45,722 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:45,735 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:48,435 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:48,447 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:49,957 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:23:50,958 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:51,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:51,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:52,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:23:52,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:23:52,796 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:23:53,992 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:54,008 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:56,365 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:23:56,677 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:56,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:59,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:23:59,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:00,801 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:01,370 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:02,224 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:02,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:04,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:04,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:06,805 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:06,976 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:07,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:24:07,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:24:07,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:07,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:10,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:10,084 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:10,807 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:12,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:12,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:12,658 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:12,711 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:24:12,712 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:24:12,712 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:24:12,713 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:24:12,808 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:24:13,809 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:14,810 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:15,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:15,317 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:16,811 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:17,687 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:17,976 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:17,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:19,960 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:24:22,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:24:22,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:24:23,400 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:25,946 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:24:27,206 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:27,223 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:28,401 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:29,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:29,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:32,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:32,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:32,823 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:33,823 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:35,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:35,275 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:37,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:24:37,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:24:37,860 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:37,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:39,405 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:40,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:40,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:42,830 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:43,126 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:43,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:44,996 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:45,815 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:45,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:48,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:48,554 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:48,834 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:49,962 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:24:50,587 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:50,647 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:24:50,648 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:24:50,648 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:24:50,648 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:24:50,836 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:24:51,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:51,288 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:51,837 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:52,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:24:52,173 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:24:52,838 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:53,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:53,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:54,839 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:56,545 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:56,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:24:56,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:56,841 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:24:59,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:24:59,228 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:01,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:01,874 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:01,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:04,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:04,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:06,558 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:07,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:25:07,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:25:08,855 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:12,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:12,375 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:25:13,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:13,694 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:16,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:16,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:17,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:18,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:19,012 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:19,964 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:25:20,864 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:21,746 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:21,764 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:22,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:25:22,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:25:22,357 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:24,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:24,414 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:26,257 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:25:26,258 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:25:26,258 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:25:26,259 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:25:26,867 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:26,868 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:25:27,061 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:27,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:27,868 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:28,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:29,870 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:29,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:32,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:32,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:32,872 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:33,578 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:34,874 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:35,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:35,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:36,880 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:37,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:25:37,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:25:37,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:37,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:39,403 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:40,444 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:40,459 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:43,031 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:43,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:44,404 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:45,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:45,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:46,889 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:25:48,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:48,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:49,422 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:49,966 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:25:51,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:51,201 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:52,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:25:52,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:25:55,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:25:59,107 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:25:59,131 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:59,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:25:59,562 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:25:59,563 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:25:59,563 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:25:59,563 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:25:59,899 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:26:00,564 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:00,900 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:01,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:01,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:01,901 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:04,391 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:04,420 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:05,565 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:07,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:07,138 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:07,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:26:07,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:26:08,907 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:09,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:09,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:11,398 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:12,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:12,598 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:12,909 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:15,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:15,126 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:16,912 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:16,914 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:17,747 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:17,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:18,914 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:19,232 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:26:19,234 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:26:19,234 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:26:19,234 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:26:19,914 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:19,914 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:26:19,969 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:26:20,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:20,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:20,915 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:21,970 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:22,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:26:22,174 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:26:23,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:23,105 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:25,753 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:25,769 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:27,353 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:28,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:28,429 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:31,134 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:31,162 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:32,354 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:33,811 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:33,836 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:36,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:36,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:37,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:26:37,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:26:37,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:40,928 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:42,763 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:44,560 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:26:45,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:45,604 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:48,101 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:48,259 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:48,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:48,934 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:49,972 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:26:50,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:50,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:52,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:26:52,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:26:53,517 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:53,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:53,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:54,939 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:56,282 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:56,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:58,531 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:26:58,890 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:58,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:26:58,942 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:26:59,401 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:26:59,402 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:26:59,403 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:26:59,403 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:26:59,942 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:27:00,943 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:01,486 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:01,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:01,944 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:02,945 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:04,081 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:04,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:04,433 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:06,772 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:06,788 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:07,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:27:07,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:27:09,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:09,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:10,402 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:11,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:11,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:14,478 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:14,509 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:15,402 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:17,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:17,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:18,957 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:19,958 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:19,975 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:27:19,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:20,976 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:22,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:27:22,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:27:22,655 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:22,680 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:26,797 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:28,965 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:30,657 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:27:30,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:30,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:31,798 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:32,967 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:33,330 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:33,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:35,960 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:35,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:36,366 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:27:36,368 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:27:36,368 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:27:36,368 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:27:36,971 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:36,971 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:27:37,173 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:37,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:27:37,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:27:37,972 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:38,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:38,562 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:40,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:41,089 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:41,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:42,514 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:42,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:43,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:43,770 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:46,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:46,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:47,515 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:49,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:49,039 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:49,977 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:27:51,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:51,709 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:52,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:27:52,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:27:53,349 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:27:54,672 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:54,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:56,986 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:27:57,375 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:57,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:27:58,908 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:00,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:00,079 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:02,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:02,772 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:03,908 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:05,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:05,515 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:06,993 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:07,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:28:07,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:28:08,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:08,131 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:09,396 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:10,712 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:28:10,713 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:28:10,714 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:28:10,714 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:28:10,996 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:28:11,998 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:12,998 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:14,715 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:15,926 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:28:15,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:15,994 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:17,002 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:18,548 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:18,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:19,004 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:19,981 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:28:19,982 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:21,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:21,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:22,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:28:22,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:28:23,007 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:23,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:23,805 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:25,404 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:26,441 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:26,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:29,086 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:29,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:30,455 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:31,013 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:31,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:31,757 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:34,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:34,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:35,456 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:36,995 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:37,015 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:37,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:28:37,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:28:39,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:39,616 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:41,329 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:41,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:28:41,681 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:28:41,681 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:28:41,682 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:28:42,020 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:28:42,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:42,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:43,021 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:44,023 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:45,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:45,076 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:46,683 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:47,773 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:47,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:49,026 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:49,982 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:28:50,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:50,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:51,983 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:52,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:28:52,175 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:28:53,161 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:53,186 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:28:55,030 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:28:57,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:28:59,034 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:01,035 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:01,637 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:29:01,637 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:29:01,638 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:29:01,638 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:29:01,656 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:29:02,036 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:29:02,639 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:02,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:02,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:03,037 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:04,038 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:05,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:05,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:07,175 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:29:07,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:29:07,700 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:07,719 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:08,369 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:10,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:10,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:13,046 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:13,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:13,370 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:15,663 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:15,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:18,371 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:18,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:18,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:19,989 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:29:21,053 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:21,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:21,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:22,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:29:22,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:29:23,404 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:23,965 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:23,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:26,602 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:26,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:29,020 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:29,059 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:29,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:29,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:31,995 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:32,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:34,400 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:34,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:34,697 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:35,068 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:37,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:29:37,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:29:37,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:37,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:39,071 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:39,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:39,945 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:39,997 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:40,050 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:29:40,052 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:29:40,052 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:29:40,052 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:29:40,072 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:29:41,073 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:42,074 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:43,075 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:29:45,055 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:47,570 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:29:47,851 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:47,866 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:49,992 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:29:50,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:50,682 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:50,994 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:52,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:29:52,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:29:53,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:53,432 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:56,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:56,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:56,399 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:29:58,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:58,767 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:29:59,089 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:01,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:01,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:02,216 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:04,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:04,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:06,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:06,838 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:07,097 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:07,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:30:07,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:30:07,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:09,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:09,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:11,953 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:11,975 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:12,575 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:13,102 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:14,586 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:14,613 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:15,624 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:30:15,625 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:30:15,625 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:30:15,626 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:30:16,105 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:30:17,106 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:17,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:17,283 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:17,627 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:18,107 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:19,108 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:19,873 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:19,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:19,994 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:30:22,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:30:22,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:30:22,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:22,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:23,112 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:23,350 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:25,394 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:25,416 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:28,351 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:33,121 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:33,804 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:30:33,834 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:33,852 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:34,008 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:36,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:36,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:37,178 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:30:37,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:30:39,214 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:39,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:39,337 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:41,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:41,475 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:43,129 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:44,014 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:44,046 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:45,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:46,955 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:46,977 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:47,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:30:47,697 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:30:47,697 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:30:47,698 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:30:48,135 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:30:49,135 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:49,717 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:49,750 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:49,997 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:30:50,137 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:50,998 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:52,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:30:52,176 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:30:52,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:52,576 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:53,139 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:55,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:55,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:56,533 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:30:57,143 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:30:57,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:57,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:30:59,145 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:00,491 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:00,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:02,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:03,182 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:03,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:05,774 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:05,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:07,134 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:07,176 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:31:07,177 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:31:08,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:08,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:09,154 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:10,989 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:11,031 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:13,029 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:18,030 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:19,078 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:31:19,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:19,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:19,999 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:31:21,050 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:31:21,051 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:31:21,051 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:31:21,051 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:31:21,164 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:31:21,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:21,762 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:22,165 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:22,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:31:22,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:31:23,166 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:23,385 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:24,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:24,451 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:27,181 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:27,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:28,855 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:29,171 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:29,830 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:29,850 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:32,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:32,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:34,493 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:34,963 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:34,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:35,176 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:37,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:31:37,177 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:31:37,206 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:37,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:39,468 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:39,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:39,838 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:41,892 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:41,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:42,183 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:43,184 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:31:44,266 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:44,287 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:45,012 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:46,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:46,713 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:49,082 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:49,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:50,001 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:31:51,002 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:31:51,598 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:51,611 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:52,178 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:31:52,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:31:54,035 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:54,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:31:56,396 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:01,397 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:01,454 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:32:02,603 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:02,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:05,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:05,059 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:06,398 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:07,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:32:07,177 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:32:07,430 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:07,446 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:09,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:09,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:11,207 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:12,016 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:12,159 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:12,176 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:13,209 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:14,525 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:14,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:15,126 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 22:32:15,126 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 22:32:15,132 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:32:15,191 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:32:15,193 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:32:15,210 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:15,211 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav +2024-05-13 22:32:15,211 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav +2024-05-13 22:32:15,211 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json +2024-05-13 22:32:15,211 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 22:32:15,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 22:32:15,822 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/rarkjo8c-media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json +2024-05-13 22:32:16,060 INFO wandb-upload_1:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmpb6vxw377 +2024-05-13 22:32:16,490 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/o45hsqba-media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav +2024-05-13 22:32:16,498 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/s563f1vp-media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav +2024-05-13 22:32:16,564 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/2u1kfney-media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav +2024-05-13 22:32:16,577 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/2ommzfqv-media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav +2024-05-13 22:32:16,594 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/buiei4kw-media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav +2024-05-13 22:32:16,600 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/0nluwo0n-media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav +2024-05-13 22:32:16,626 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/19s5d7id-media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav +2024-05-13 22:32:16,673 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep762 - {'id': 'QXJ0aWZhY3Q6ODM0ODY2MDEy', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI5MjU4Ng==', 'latestArtifact': None}} +2024-05-13 22:32:16,673 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,674 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json with policy now +2024-05-13 22:32:16,674 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:32:16,674 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:32:16,675 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:32:16,675 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,675 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav with policy now +2024-05-13 22:32:16,675 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,675 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav with policy now +2024-05-13 22:32:16,675 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,675 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav with policy now +2024-05-13 22:32:16,676 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,676 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav with policy now +2024-05-13 22:32:16,676 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,676 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav with policy now +2024-05-13 22:32:16,676 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,676 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav with policy now +2024-05-13 22:32:16,676 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:32:16,676 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav with policy now +2024-05-13 22:32:17,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:17,075 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:17,105 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/4b938q0k-media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json +2024-05-13 22:32:17,212 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:32:17,673 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/chepain9-media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav +2024-05-13 22:32:17,677 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:17,759 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/r8eup6fz-media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav +2024-05-13 22:32:17,765 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/eosarlqk-media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav +2024-05-13 22:32:17,798 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/dgytxsr7-media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav +2024-05-13 22:32:17,812 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/bzf499sl-media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav +2024-05-13 22:32:17,815 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/zsi7xr2u-media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav +2024-05-13 22:32:17,816 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/hc5iidyo-media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav +2024-05-13 22:32:18,213 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:19,214 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:19,730 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:19,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:20,004 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:32:21,217 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:22,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:32:22,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:32:22,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:22,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:23,218 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:23,378 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:23,842 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:32:23,844 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:32:23,844 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:32:23,844 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:32:24,219 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:32:24,943 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:24,967 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:25,220 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:26,221 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:27,709 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:27,736 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:28,845 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:30,367 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:30,394 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:33,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:33,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:33,846 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:35,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:35,982 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:37,177 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:32:37,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:32:38,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:38,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:39,314 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:41,234 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:44,882 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:45,237 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:46,847 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:32:46,889 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:46,901 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:47,239 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:32:49,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:49,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:49,978 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:50,006 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:32:52,178 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:32:52,178 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:32:52,571 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:52,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:55,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:55,358 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:55,366 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:32:58,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:32:58,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:00,367 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:00,720 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:00,755 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:03,454 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:03,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:05,368 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:06,061 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:06,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:07,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:33:07,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:33:07,256 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:08,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:08,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:10,758 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:10,775 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:11,338 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:13,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:13,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:13,375 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:33:13,376 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:33:13,376 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:33:13,376 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:33:14,261 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:14,262 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:33:15,262 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:15,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:15,470 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:16,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:17,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:17,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:20,008 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:33:20,398 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:20,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:21,267 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:22,009 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:22,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:33:22,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:33:22,873 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:22,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:23,268 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:27,272 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:27,686 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:29,274 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:31,195 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:33:31,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:31,252 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:32,687 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:33,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:33,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:36,178 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:36,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:37,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:33:37,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:33:38,394 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:38,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:38,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:41,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:41,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:43,395 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:43,873 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:33:43,875 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:33:43,875 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:33:43,875 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:33:43,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:43,930 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:44,289 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:33:45,290 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:46,291 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:46,689 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:46,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:48,876 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:49,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:49,608 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:50,011 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:33:52,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:33:52,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:33:52,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:52,383 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:54,833 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:33:55,062 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:55,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:55,299 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:33:57,657 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:33:57,679 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:00,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:00,317 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:00,454 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:01,304 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:02,951 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:02,982 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:05,307 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:05,508 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:05,838 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:05,862 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:07,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:34:07,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:34:07,309 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:07,751 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:34:07,753 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:34:07,753 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:34:07,753 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:34:08,310 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:34:08,675 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:08,697 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:09,311 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:10,312 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:10,754 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:15,755 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:17,588 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:34:17,618 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:17,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:20,013 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:34:20,590 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:20,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:21,014 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:21,320 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:22,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:34:22,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:34:23,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:23,570 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:26,370 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:26,450 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:26,485 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:29,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:29,374 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:31,524 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:32,227 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:32,252 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:33,329 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:35,200 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:35,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:36,529 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:37,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:34:37,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:34:38,174 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:38,197 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:39,333 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:41,177 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:41,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:42,522 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:43,336 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:43,977 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:44,000 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:44,834 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:34:44,835 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:34:44,835 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:34:44,835 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:34:45,338 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:34:46,338 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:46,951 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:46,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:47,339 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:48,128 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:49,341 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:49,467 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:49,487 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:50,018 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:34:52,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:34:52,180 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:34:52,434 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:52,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:53,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:34:55,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:55,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:57,348 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:34:57,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:57,875 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:34:59,140 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:04,141 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:06,179 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:35:06,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:06,809 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:07,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:35:07,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:35:09,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:09,073 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:09,606 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:11,360 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:12,064 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:12,088 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:14,606 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:15,072 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:15,093 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:17,365 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:18,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:18,037 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:20,021 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:35:20,023 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:20,948 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:20,968 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:21,521 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:35:21,522 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:35:21,522 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:35:21,522 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:35:22,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:35:22,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:35:22,369 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:35:23,370 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:23,881 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:23,906 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:24,371 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:25,923 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:26,794 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:26,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:27,373 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:29,375 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:29,684 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:29,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:31,354 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:31,377 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:32,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:32,778 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:35,765 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:35,786 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:36,355 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:37,180 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:35:37,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:35:38,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:38,818 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:41,372 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:41,776 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:41,804 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:44,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:44,749 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:46,373 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:47,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:47,704 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:49,392 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:50,024 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:35:52,025 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:52,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:35:52,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:35:55,167 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:35:55,168 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:35:55,168 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:35:55,169 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:35:55,397 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:35:56,398 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:56,451 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:35:56,477 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:35:57,169 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:35:57,399 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:35:57,833 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:00,776 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:00,797 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:02,496 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:03,405 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:03,719 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:03,736 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:06,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:06,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:07,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:36:07,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:36:07,408 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:08,396 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:09,410 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:09,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:09,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:11,412 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:12,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:12,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:14,190 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:15,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:15,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:18,453 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:18,468 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:19,191 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:20,026 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:36:21,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:21,437 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:22,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:36:22,181 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:36:24,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:24,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:24,394 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:25,871 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:36:25,871 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:36:25,872 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:36:25,872 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:36:26,425 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:36:27,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:27,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:27,425 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:28,426 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:29,873 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:30,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:30,248 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:33,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:33,209 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:34,874 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:36,158 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:36,179 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:37,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:36:37,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:36:37,435 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:39,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:39,129 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:40,340 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:43,441 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:46,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:47,444 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:48,061 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:36:48,083 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:49,444 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:49,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:50,029 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:36:50,406 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:36:50,407 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:36:50,408 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:36:50,408 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:36:50,447 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:36:51,409 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:52,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:36:52,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:36:52,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:52,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:52,449 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:53,450 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:36:55,390 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:55,416 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:57,388 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:36:58,374 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:36:58,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:01,327 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:01,358 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:02,390 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:04,392 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:04,409 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:05,460 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:07,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:37:07,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:37:07,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:07,360 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:08,381 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:10,273 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:10,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:13,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:13,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:13,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:16,271 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:16,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:17,472 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:18,814 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:19,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:19,311 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:20,030 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:37:22,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:37:22,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:37:22,339 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:22,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:23,479 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:24,356 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:25,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:25,317 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:27,482 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:28,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:28,336 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:29,102 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:37:29,103 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:37:29,103 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:37:29,104 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:37:29,484 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:37:30,104 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:30,485 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:31,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:31,235 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:31,486 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:33,488 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:35,430 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:37,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:37:37,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:37:39,991 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:37:40,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:40,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:41,406 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:43,006 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:43,030 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:43,498 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:46,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:46,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:46,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:49,067 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:49,088 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:50,033 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:37:52,034 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:52,150 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:52,173 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:52,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:37:52,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:37:55,260 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:55,280 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:55,509 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:37:57,730 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:37:58,193 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:37:58,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:01,198 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:01,217 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:03,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:03,517 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:04,140 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:04,174 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:05,241 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:38:05,242 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:38:05,242 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:38:05,243 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:38:05,519 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:38:06,520 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:07,178 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:07,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:38:07,182 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:38:07,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:07,521 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:08,794 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:10,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:10,193 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:11,525 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:13,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:13,129 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:13,526 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:14,382 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:15,528 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:16,048 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:16,077 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:19,029 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:19,050 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:19,383 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:20,035 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:38:22,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:22,082 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:22,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:38:22,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:38:25,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:30,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:30,896 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:38:30,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:30,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:33,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:33,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:35,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:35,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:36,843 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:36,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:37,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:38:37,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:38:39,822 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:39,847 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:41,403 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:41,954 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:38:41,955 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:38:41,956 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:38:41,956 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:38:42,551 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:38:42,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:42,923 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:43,552 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:44,553 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:45,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:45,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:46,957 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:48,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:48,981 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:49,558 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:50,038 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:38:51,898 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:51,912 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:52,100 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:52,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:38:52,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:38:53,561 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:54,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:54,855 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:55,563 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:57,187 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:38:57,564 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:38:57,811 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:38:57,833 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:00,653 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:00,681 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:02,190 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:03,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:03,624 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:06,701 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:06,713 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:07,182 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:39:07,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:39:07,345 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:09,685 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:09,704 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:12,346 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:12,671 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:12,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:12,834 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:39:12,835 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:39:12,835 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:39:12,835 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:39:13,579 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:39:14,579 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:15,580 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:17,836 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:20,040 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:39:21,543 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:39:21,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:21,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:22,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:39:22,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:39:23,333 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:23,587 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:24,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:24,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:27,417 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:27,428 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:28,420 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:29,592 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:30,346 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:30,367 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:33,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:33,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:33,458 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:33,595 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:36,300 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:36,312 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:36,582 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:39:36,583 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:39:36,583 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:39:36,583 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:39:36,598 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:39:37,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:39:37,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:39:37,599 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:38,600 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:39,233 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:39,245 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:39,356 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:39,600 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:42,239 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:42,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:44,357 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:45,299 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:45,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:48,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:48,315 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:49,358 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:50,042 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:39:51,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:51,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:51,611 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:39:52,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:39:52,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:39:54,318 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:54,332 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:54,372 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:39:57,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:57,308 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:39:59,374 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:00,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:00,267 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:03,328 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:03,342 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:03,622 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:04,880 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:07,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:40:07,183 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:40:10,374 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:40:10,415 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:10,444 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:10,649 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:11,629 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:13,423 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:13,443 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:15,632 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:15,860 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:15,894 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:16,394 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:16,821 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:40:16,822 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:40:16,823 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:40:16,823 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:40:17,634 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:17,634 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:40:18,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:18,879 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:19,636 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:20,045 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:40:21,637 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:21,882 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:21,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:22,046 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:22,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:40:22,184 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:40:24,827 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:24,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:27,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:27,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:27,772 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:30,692 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:30,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:31,646 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:32,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:33,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:33,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:36,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:36,665 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:37,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:40:37,184 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:40:38,314 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:39,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:39,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:41,654 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:42,647 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:42,668 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:43,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:45,653 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:45,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:48,703 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:48,722 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:49,096 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:49,660 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:50,047 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:40:51,652 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:51,678 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:40:51,982 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:40:51,983 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:40:51,983 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:40:51,983 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:40:52,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:40:52,184 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:40:52,663 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:40:53,664 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:54,372 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:40:57,668 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:59,670 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:40:59,766 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:00,243 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:41:00,282 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:00,293 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:01,671 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:03,158 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:03,172 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:04,768 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:06,150 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:06,174 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:07,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:41:07,184 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:41:09,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:09,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:10,404 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:12,190 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:12,213 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:15,229 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:15,248 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:15,406 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:18,219 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:18,240 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:20,049 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:41:21,051 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:21,254 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:21,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:21,686 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:22,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:41:22,184 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:41:24,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:24,247 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:26,310 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:27,062 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:41:27,063 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:41:27,063 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:41:27,063 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:41:27,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:27,219 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:27,691 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:27,691 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:41:29,693 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:30,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:30,133 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:32,064 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:33,122 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:33,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:35,698 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:36,078 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:36,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:37,181 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:37,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:41:37,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:41:37,700 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:39,103 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:39,132 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:41,703 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:42,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:42,154 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:42,993 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:43,705 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:47,994 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:50,052 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:41:51,124 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:41:51,152 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:51,166 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:52,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:41:52,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:41:53,377 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:53,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:54,010 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:57,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:57,078 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:58,292 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:41:58,293 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:41:58,293 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:41:58,294 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:41:58,717 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:41:59,294 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:41:59,718 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:41:59,975 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:41:59,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:02,930 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:02,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:04,296 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:05,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:05,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:07,184 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:42:07,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:42:08,901 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:08,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:09,440 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:09,726 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:11,907 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:11,929 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:14,904 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:14,930 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:15,090 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:15,731 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:17,986 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:18,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:19,735 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:20,054 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:42:20,254 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:20,911 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:20,934 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:21,737 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:22,185 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:42:22,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:42:22,555 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:42:23,237 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:42:23,237 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:42:23,238 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:42:23,738 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:23,739 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:42:23,810 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:23,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:25,741 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:26,239 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:26,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:26,745 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:29,631 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:29,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:31,240 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:32,607 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:32,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:36,691 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:37,185 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:42:37,185 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:42:37,751 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:41,157 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:42:41,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:41,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:42,326 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:44,727 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:44,743 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:47,733 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:47,760 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:47,772 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:48,148 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:50,057 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:42:50,738 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:50,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:52,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:42:52,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:42:53,607 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:53,784 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:53,814 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:55,766 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:42:56,734 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:56,756 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:59,077 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:42:59,645 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:59,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:42:59,768 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:00,386 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:43:00,387 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:43:00,387 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:43:00,388 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:43:00,769 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:43:01,770 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:02,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:02,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:04,758 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:05,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:05,555 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:05,773 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:07,185 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:43:07,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:43:08,485 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:08,502 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:10,368 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:11,557 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:11,577 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:13,778 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:14,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:14,566 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:16,110 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:17,564 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:17,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:20,059 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:43:20,555 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:20,579 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:22,060 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:22,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:43:22,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:43:23,540 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:23,558 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:25,787 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:27,970 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:31,791 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:32,001 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:43:32,406 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:32,417 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:33,645 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:34,235 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:43:34,236 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:43:34,236 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:43:34,237 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:43:34,793 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:43:35,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:35,381 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:35,794 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:37,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:43:37,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:43:38,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:38,341 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:39,523 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:39,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:41,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:41,248 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:41,799 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:43,801 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:43:44,294 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:44,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:44,818 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:47,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:47,296 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:49,818 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:50,061 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:43:50,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:50,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:52,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:43:52,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:43:53,300 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:53,333 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:55,400 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:43:55,999 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:56,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:58,660 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:43:58,693 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:00,402 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:01,614 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:01,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:03,819 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:04,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:04,659 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:06,061 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:07,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:44:07,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:44:07,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:07,597 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:09,385 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:44:09,386 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:44:09,386 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:44:09,386 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:44:09,823 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:09,823 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:44:10,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:10,555 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:11,387 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:11,826 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:13,479 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:13,495 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:16,721 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:17,831 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:20,063 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:44:21,835 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:22,021 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:22,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:44:22,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:44:22,199 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:44:22,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:22,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:23,837 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:25,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:25,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:25,838 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:27,038 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:28,095 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:28,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:31,127 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:31,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:32,039 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:34,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:34,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:37,041 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:37,144 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:37,162 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:37,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:44:37,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:44:40,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:40,119 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:44:40,120 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:44:40,120 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:44:40,121 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:44:40,139 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:40,852 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:44:41,853 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:42,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:43,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:43,082 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:46,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:46,114 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:47,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:49,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:49,147 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:50,066 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:44:51,862 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:52,120 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:52,143 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:52,183 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:52,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:44:52,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:44:55,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:55,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:57,867 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:44:58,041 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:44:58,052 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:44:58,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:00,987 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:01,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:01,870 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:03,174 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:03,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:03,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:05,467 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:45:05,468 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:45:05,469 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:45:05,469 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:45:05,874 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:05,875 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:45:07,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:45:07,186 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:45:07,877 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:08,378 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:12,872 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:45:12,903 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:13,380 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:14,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:17,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:17,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:18,560 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:19,888 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:20,069 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:45:20,368 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:20,379 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:22,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:45:22,187 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:45:23,328 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:23,340 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:24,318 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:26,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:26,309 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:29,256 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:29,267 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:29,897 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:30,232 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:32,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:32,230 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:35,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:35,242 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:35,729 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:37,187 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:45:37,187 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:45:37,904 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:38,286 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:38,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:41,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:41,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:41,597 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:41,908 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:43,048 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:45:43,049 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:45:43,050 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:45:43,050 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:45:43,910 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:43,910 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:45:44,238 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:44,250 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:45,911 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:47,251 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:47,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:47,354 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:47,913 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:50,070 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:45:50,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:50,298 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:52,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:45:52,187 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:45:52,337 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:52,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:53,331 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:45:55,241 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:55,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:45:57,922 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:45:59,285 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:02,191 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:46:03,665 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:03,702 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:04,286 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:06,735 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:06,757 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:07,186 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:46:07,187 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:46:09,487 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:09,888 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:09,913 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:10,934 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:12,928 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:12,940 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:14,487 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:15,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:16,011 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:16,939 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:18,663 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:18,674 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:20,074 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:46:20,075 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:21,348 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:21,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:21,944 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:22,688 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:46:22,688 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:46:24,080 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:24,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:24,946 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:25,239 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:26,753 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:26,765 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:29,495 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:29,505 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:30,240 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:32,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:32,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:35,240 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:35,310 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:35,322 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:37,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:46:37,195 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:46:38,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:38,112 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:40,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:40,424 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:40,687 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:43,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:43,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:45,688 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:50,075 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:46:50,969 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:51,018 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:51,856 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:46:51,890 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:51,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:52,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:46:52,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:46:52,971 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:54,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:54,715 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:54,972 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:55,450 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 22:46:55,450 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 22:46:55,456 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:46:55,524 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:46:55,526 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_4871014c416be45744ef.wav +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_819325490f684d367b39.wav +2024-05-13 22:46:55,974 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav +2024-05-13 22:46:55,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json +2024-05-13 22:46:55,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav +2024-05-13 22:46:55,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 22:46:55,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 22:46:56,405 INFO wandb-upload_1:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmp0gy_pukm +2024-05-13 22:46:56,648 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/j9b5d3n0-media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json +2024-05-13 22:46:57,004 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep889 - {'id': 'QXJ0aWZhY3Q6ODM0ODg2Mjc5', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI5ODg3OQ==', 'latestArtifact': None}} +2024-05-13 22:46:57,005 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,005 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json with policy now +2024-05-13 22:46:57,005 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:46:57,005 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:46:57,006 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:46:57,006 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:46:57,007 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,007 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav with policy now +2024-05-13 22:46:57,007 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,007 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_4871014c416be45744ef.wav with policy now +2024-05-13 22:46:57,007 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,007 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav with policy now +2024-05-13 22:46:57,007 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,009 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav with policy now +2024-05-13 22:46:57,009 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,009 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_819325490f684d367b39.wav with policy now +2024-05-13 22:46:57,009 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,009 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav with policy now +2024-05-13 22:46:57,010 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 22:46:57,010 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav with policy now +2024-05-13 22:46:57,093 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/tn4irtcn-media/audio/Speech samples/eval_889_4871014c416be45744ef.wav +2024-05-13 22:46:57,198 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/y0nfu36w-media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav +2024-05-13 22:46:57,219 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/yxbqme64-media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav +2024-05-13 22:46:57,283 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/nxgqgrbc-media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav +2024-05-13 22:46:57,289 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/exuljp42-media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav +2024-05-13 22:46:57,298 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/wij34dh8-media/audio/Speech samples/eval_889_819325490f684d367b39.wav +2024-05-13 22:46:57,307 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/0xe8ygpj-media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav +2024-05-13 22:46:57,433 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/xxf7p9fk-media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json +2024-05-13 22:46:57,565 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:57,584 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:46:57,975 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:57,976 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:46:58,243 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/x6jox6cv-media/audio/Speech samples/eval_889_4871014c416be45744ef.wav +2024-05-13 22:46:58,350 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/1h918ijy-media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav +2024-05-13 22:46:58,355 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/yb9abl1z-media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav +2024-05-13 22:46:58,433 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/75kqlmkv-media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav +2024-05-13 22:46:58,462 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/km3jn9e7-media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav +2024-05-13 22:46:58,977 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:46:59,323 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/nu83r2us-media/audio/Speech samples/eval_889_819325490f684d367b39.wav +2024-05-13 22:46:59,464 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/1kwi9m8p-media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav +2024-05-13 22:47:00,243 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:00,264 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:01,086 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:47:01,087 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:47:01,088 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:47:01,088 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:47:01,980 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:01,980 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:47:02,088 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:02,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:02,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:02,981 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:05,499 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:05,523 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:06,985 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:07,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:47:07,195 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:47:07,350 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:08,102 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:08,128 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:08,987 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:10,556 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:10,581 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:10,989 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:12,755 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:13,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:13,327 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:16,137 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:16,155 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:17,755 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:18,893 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:18,918 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:20,078 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:47:21,667 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:21,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:22,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:47:22,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:47:23,693 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:24,433 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:24,459 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:27,005 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:27,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:27,232 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:29,515 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:29,978 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:30,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:30,962 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:47:30,964 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:47:30,964 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:47:30,964 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:47:31,009 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:47:32,010 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:33,011 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:35,013 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:35,339 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:37,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:47:37,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:47:39,766 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:47:40,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:40,510 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:40,693 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:43,410 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:43,433 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:45,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:46,105 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:46,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:48,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:49,019 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:50,080 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:47:51,082 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:51,576 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:51,589 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:52,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:47:52,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:47:54,225 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:54,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:56,704 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:47:56,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:56,915 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:57,035 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:47:59,513 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:47:59,526 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:01,706 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:02,180 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:02,191 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:03,042 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:04,854 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:04,865 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:07,195 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:48:07,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:48:07,363 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:07,648 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:07,746 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:10,251 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:10,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:11,050 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:12,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:12,620 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:48:12,620 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:48:12,620 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:48:12,621 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:48:13,052 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:48:13,378 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:13,394 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:14,053 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:15,054 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:15,932 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:15,942 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:17,056 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:18,393 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:18,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:18,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:19,058 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:20,082 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:48:22,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:48:22,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:48:23,693 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:26,364 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:48:26,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:26,407 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:28,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:28,949 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:28,959 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:31,601 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:31,610 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:34,064 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:34,156 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:34,167 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:35,072 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:36,707 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:36,719 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:37,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:48:37,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:48:39,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:39,472 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:39,693 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:42,045 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:42,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:43,079 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:44,695 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:44,711 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:45,287 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:46,876 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:48:46,877 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:48:46,877 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:48:46,878 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:48:47,083 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:48:47,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:47,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:48,084 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:49,085 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:49,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:49,919 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:50,085 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:48:50,680 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:52,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:48:52,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:48:52,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:52,549 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:53,091 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:55,093 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:55,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:55,111 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:56,544 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:48:57,095 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:48:57,625 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:48:57,635 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:00,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:00,405 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:01,546 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:02,996 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:03,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:06,546 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:07,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:49:07,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:49:09,105 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:10,827 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:49:10,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:10,870 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:12,290 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:13,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:13,465 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:16,031 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:16,044 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:17,291 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:18,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:18,724 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:19,412 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:49:19,413 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:49:19,413 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:49:19,414 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:49:20,088 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:49:20,114 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:20,114 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:49:21,115 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:21,345 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:21,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:22,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:49:22,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:49:22,362 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:23,944 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:23,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:27,120 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:27,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:27,791 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:28,143 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:30,350 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:30,362 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:33,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:33,019 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:33,125 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:33,954 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:35,127 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:35,559 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:35,572 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:37,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:49:37,196 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:49:38,105 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:38,118 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:39,131 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:39,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:40,735 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:40,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:43,352 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:43,363 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:44,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:45,935 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:49:45,936 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:49:45,936 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:49:45,936 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:49:45,962 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:45,974 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:46,137 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:49:47,138 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:48,139 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:48,620 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:48,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:49,937 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:50,091 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:49:52,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:49:52,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:49:55,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:49:56,767 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:49:56,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:56,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:59,148 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:49:59,354 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:49:59,370 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:01,518 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:01,937 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:01,948 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:04,737 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:04,751 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:07,029 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:07,156 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:07,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:50:07,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:50:07,313 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:07,326 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:10,374 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:10,387 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:12,376 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:12,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:12,945 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:13,161 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:15,712 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:15,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:17,165 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:17,611 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:17,661 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:50:17,662 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:50:17,662 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:50:17,662 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:50:18,166 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:18,166 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:50:19,106 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:19,117 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:19,166 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:20,095 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:50:21,168 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:21,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:21,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:22,196 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:50:22,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:50:22,694 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:24,589 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:24,600 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:27,192 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:27,205 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:27,695 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:29,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:29,884 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:32,468 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:32,480 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:32,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:35,018 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:35,030 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:37,197 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:50:37,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:50:38,695 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:41,185 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:43,722 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:43,916 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:50:44,314 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:44,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:46,972 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:46,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:47,190 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:49,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:49,546 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:49,571 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:50,095 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:50:52,197 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:50:52,197 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:50:52,338 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:52,349 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:53,195 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:54,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:50:54,955 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:54,966 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:55,561 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:50:55,562 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:50:55,562 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:50:55,563 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:50:56,199 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:56,199 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:50:57,200 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:50:57,611 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:57,622 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:50:59,202 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:00,120 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:00,169 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:00,181 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:01,204 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:02,665 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:02,683 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:05,122 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:05,242 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:05,262 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:07,197 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:51:07,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:51:08,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:08,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:10,695 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:11,030 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:11,044 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:13,755 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:13,765 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:15,713 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:16,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:16,393 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:17,221 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:18,990 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:19,001 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:20,097 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:51:21,099 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:21,680 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:21,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:22,197 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:51:22,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:51:25,230 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:26,630 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:29,469 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:51:29,470 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:51:29,470 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:51:29,471 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:51:29,605 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:51:30,093 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:30,108 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:30,235 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:30,235 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:51:31,236 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:32,472 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:32,581 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:32,593 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:35,152 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:35,163 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:35,240 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:37,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:51:37,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:51:37,242 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:37,695 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:37,889 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:37,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:39,244 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:40,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:40,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:43,028 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:43,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:43,588 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:45,585 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:45,599 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:48,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:48,274 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:48,588 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:50,100 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:51:50,850 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:50,862 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:52,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:51:52,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:51:53,257 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:51:53,421 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:53,434 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:53,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:56,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:56,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:58,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:51:58,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:51:58,766 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:01,783 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:01,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:02,121 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:52:02,122 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:52:02,123 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:52:02,123 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:52:02,265 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:52:03,266 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:04,124 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:04,267 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:04,385 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:04,396 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:07,183 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:07,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:07,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:52:07,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:52:09,229 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:09,272 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:14,358 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:52:14,645 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:15,203 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:15,215 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:15,278 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:17,280 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:17,957 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:18,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:19,282 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:20,102 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:52:20,105 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:20,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:20,232 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:22,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:52:22,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:52:22,961 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:22,971 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:25,189 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:25,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:25,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:27,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:27,610 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:28,699 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:52:28,700 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:52:28,700 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:52:28,701 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:52:29,291 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:29,292 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:52:30,038 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:30,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:30,292 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:30,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:32,304 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:32,319 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:34,638 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:34,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:35,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:36,887 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:36,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:37,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:52:37,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:52:39,258 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:39,270 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:41,615 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:41,620 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:41,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:43,304 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:43,922 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:43,933 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:46,307 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:46,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:46,622 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:48,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:48,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:49,309 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:50,105 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:52:52,106 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:52,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:52:52,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:52:55,316 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:52:55,942 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:52:55,980 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:55,998 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:57,215 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:52:58,311 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:58,324 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:52:59,319 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:00,488 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:53:00,489 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:53:00,489 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:53:00,489 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:53:00,571 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:00,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:01,321 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:01,321 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:53:02,322 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:02,323 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:02,757 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:02,771 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:03,323 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:05,058 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:05,068 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:07,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:53:07,198 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:53:07,356 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:07,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:07,369 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:09,698 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:09,708 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:12,561 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:12,575 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:12,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:14,828 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:14,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:17,452 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:17,465 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:17,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:20,108 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:53:20,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:20,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:21,339 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:22,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:53:22,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:53:22,789 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:22,801 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:23,696 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:25,400 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:25,411 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:28,005 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:28,016 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:28,801 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:29,346 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:30,595 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:30,605 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:33,941 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:35,351 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:36,873 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:53:36,874 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:53:36,874 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:53:36,874 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:53:37,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:53:37,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:53:37,354 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:53:38,354 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:39,356 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:39,534 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:53:39,575 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:40,149 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:40,160 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:41,357 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:42,640 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:42,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:43,359 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:53:45,255 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:45,268 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:45,571 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:47,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:47,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:50,111 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:53:50,941 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:50,954 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:51,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:52,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:53:52,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:53:53,514 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:53,527 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:56,220 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:56,232 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:56,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:53:58,797 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:58,813 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:53:59,375 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:01,356 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:01,376 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:01,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:04,347 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:04,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:06,941 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:06,956 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:07,087 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:07,198 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:54:07,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:54:07,388 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:09,519 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:09,530 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:11,629 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:54:11,630 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:54:11,630 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:54:11,631 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:54:12,070 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:12,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:12,393 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:54:12,631 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:13,394 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:14,395 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:15,023 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:15,037 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:17,344 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:17,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:17,398 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:18,316 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:19,399 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:20,112 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:54:21,401 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:22,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:54:22,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:54:23,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:25,580 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:54:25,610 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:25,626 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:28,358 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:28,369 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:28,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:30,891 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:30,902 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:33,099 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:33,109 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:34,337 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:35,414 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:35,687 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:35,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:37,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:54:37,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:54:38,529 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:38,543 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:39,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:41,069 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:41,081 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:43,688 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:43,699 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:44,142 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:54:44,143 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:54:44,143 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:54:44,144 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:54:44,422 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:54:45,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:45,423 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:46,271 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:46,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:46,424 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:48,825 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:48,835 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:50,115 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:54:50,253 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:51,427 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:51,845 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:51,859 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:52,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:54:52,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:54:54,431 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:54,445 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:55,431 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:55,501 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:54:57,054 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:57,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:59,434 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:54:59,686 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:54:59,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:00,837 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:01,436 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:02,799 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:02,820 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:05,837 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:07,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:55:07,199 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:55:09,624 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:55:09,625 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:55:09,625 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:55:09,626 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:55:10,446 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:55:11,255 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:55:11,447 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:11,626 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:11,949 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:11,964 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:12,448 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:14,939 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:14,950 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:16,627 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:17,587 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:17,597 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:20,117 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:55:20,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:20,142 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:22,119 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:22,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:55:22,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:55:22,790 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:22,808 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:23,458 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:25,649 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:25,664 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:27,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:28,236 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:28,249 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:30,823 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:30,837 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:31,466 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:32,881 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:33,489 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:33,504 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:35,470 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:36,072 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:36,515 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:37,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:55:37,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:55:38,697 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:39,013 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:39,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:39,474 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:39,981 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:55:39,981 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:55:39,982 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:55:39,982 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:55:40,475 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:55:41,476 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:41,521 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:41,532 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:42,477 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:43,478 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:55:43,884 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:44,136 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:44,149 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:46,768 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:46,779 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:48,885 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:49,388 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:49,829 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:50,120 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:55:52,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:55:52,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:55:54,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:55:57,279 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:55:57,666 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:55:57,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:00,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:00,277 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:00,678 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:01,494 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:02,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:02,861 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:05,679 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:05,827 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:05,840 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:07,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:56:07,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:56:08,480 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:08,491 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:09,502 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:10,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:11,122 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:11,135 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:13,759 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:13,774 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:15,507 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:15,777 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:16,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:16,736 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:16,805 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:56:16,805 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:56:16,806 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:56:16,806 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:56:17,509 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:56:18,510 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:19,263 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:19,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:19,511 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:20,122 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:56:21,123 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:21,513 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:21,780 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:21,798 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:22,199 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:56:22,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:56:23,515 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:24,323 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:24,334 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:26,698 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:26,969 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:26,990 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:29,964 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:29,983 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:31,700 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:32,534 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:32,544 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:35,092 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:35,101 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:36,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:37,200 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:56:37,200 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:56:39,531 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:41,949 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:43,475 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:56:44,483 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:44,494 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:46,950 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:47,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:47,152 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:49,540 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:49,714 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:49,730 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:50,125 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:56:52,126 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:52,200 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:56:52,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:56:52,711 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:52,723 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:53,060 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:56:53,062 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:56:53,062 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:56:53,062 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:56:53,543 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:56:54,544 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:55,357 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:55,372 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:55,545 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:56:57,169 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:56:58,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:58,031 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:56:59,548 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:00,599 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:00,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:01,551 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:02,230 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:03,553 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:03,629 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:03,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:06,261 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:06,272 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:07,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:57:07,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:57:07,367 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:08,820 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:08,831 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:11,463 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:11,474 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:12,699 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:14,538 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:14,552 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:17,121 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:17,130 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:17,567 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:18,524 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:19,691 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:19,701 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:20,126 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:57:22,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:57:22,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:57:22,221 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:22,231 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:23,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:27,070 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:57:27,071 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:57:27,071 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:57:27,072 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:57:27,576 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:57:28,577 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:29,073 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:29,578 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:31,255 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:57:31,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:31,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:34,249 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:34,367 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:34,380 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:35,583 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:37,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:57:37,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:57:37,436 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:37,449 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:39,587 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:39,822 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:40,053 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:40,064 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:42,650 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:42,670 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:43,591 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:45,264 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:45,285 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:45,297 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:45,593 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:47,896 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:47,909 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:50,130 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:57:50,591 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:50,844 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:51,131 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:52,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:57:52,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:57:52,239 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:57:52,358 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:57:52,359 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:57:52,359 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:57:52,598 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:57:53,569 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:53,581 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:53,600 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:54,601 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:57:56,175 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:56,190 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:56,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:57:58,739 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:57:58,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:01,401 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:01,420 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:01,703 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:04,007 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:04,020 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:05,610 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:06,812 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:07,040 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:07,052 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:07,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:58:07,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:58:09,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:09,621 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:11,616 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:12,557 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:17,622 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:17,915 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:58:18,333 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:19,115 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:19,706 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:20,131 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:58:21,477 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:58:21,478 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:58:21,478 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:58:21,478 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:58:21,625 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:21,625 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:58:22,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:58:22,201 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:58:22,885 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:22,895 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:23,336 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:23,626 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:24,627 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:25,443 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:25,455 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:25,628 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:28,057 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:28,068 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:28,337 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:30,639 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:30,654 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:33,338 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:33,630 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:33,641 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:36,196 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:36,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:37,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:58:37,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:58:38,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:38,744 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:38,754 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:41,295 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:41,306 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:43,644 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:44,028 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:44,043 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:44,146 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:46,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:46,633 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:49,147 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:49,291 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:49,302 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:50,135 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:58:51,650 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:58:51,869 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:51,879 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:52,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:58:52,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:58:54,518 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:54,536 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:54,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:58:57,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:57,125 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:58:57,654 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:00,399 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:00,521 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:59:00,522 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:59:00,522 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:59:00,522 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:59:00,657 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:59:01,658 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:02,660 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:04,937 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:59:04,959 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:05,533 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:05,664 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:06,393 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:07,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:59:07,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:59:07,667 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:08,669 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:08,680 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:11,391 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:11,402 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:11,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:14,009 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:14,024 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:16,628 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:16,637 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:16,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:19,194 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:19,207 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:20,138 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:59:21,752 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:21,763 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:22,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:59:22,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:59:22,364 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:23,680 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:24,461 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:24,471 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:27,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:27,110 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:27,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:29,738 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:29,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:31,689 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:32,309 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:32,320 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:33,434 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:34,936 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:34,947 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:36,282 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 22:59:36,283 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 22:59:36,283 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 22:59:36,283 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 22:59:36,693 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 22:59:37,201 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:59:37,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:59:37,581 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:37,592 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:37,693 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:38,694 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:38,701 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:40,128 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:40,141 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:41,697 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:42,676 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:42,688 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:44,496 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:45,701 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:47,702 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 22:59:49,765 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:50,139 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 22:59:51,610 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 22:59:51,642 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:51,651 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:52,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 22:59:52,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 22:59:54,281 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:54,291 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:55,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 22:59:56,849 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:56,860 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:59,514 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 22:59:59,524 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:01,258 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:01,715 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:02,087 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:02,098 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:04,771 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:04,785 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:06,259 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:07,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:00:07,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:00:07,355 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:07,366 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:09,322 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 23:00:09,323 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 23:00:09,323 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 23:00:09,323 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:00:09,722 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:00:10,025 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:10,036 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:10,722 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:11,324 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:11,723 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:12,596 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:12,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:15,199 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:15,210 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:16,390 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:17,729 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:17,853 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:17,868 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:20,142 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 23:00:20,413 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:20,425 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:21,732 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:22,144 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:22,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:00:22,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:00:23,045 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:23,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:25,627 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:25,638 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:25,736 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:27,229 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:27,738 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:28,289 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:28,302 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:32,230 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:35,829 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 23:00:35,830 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 23:00:35,831 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 23:00:35,831 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:00:36,073 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 23:00:36,097 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:36,747 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:00:36,997 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:37,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:00:37,202 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:00:37,348 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:37,748 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:38,749 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:39,930 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:39,945 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:42,573 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:42,582 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:42,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:45,149 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:45,165 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:47,704 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:47,713 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:47,725 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:50,113 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:50,124 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:50,145 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 23:00:51,760 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:52,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:00:52,203 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:00:52,403 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:52,412 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:53,702 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:54,738 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:54,748 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:56,764 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:56,975 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:56,985 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:57,765 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:00:58,832 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:00:59,193 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:00:59,202 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:01,609 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:01,619 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:01,769 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:03,771 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:03,905 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:03,920 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:04,740 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:06,292 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:06,303 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:07,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:01:07,203 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:01:08,644 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:08,656 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:10,703 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:11,056 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:11,065 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:15,704 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:18,749 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 23:01:19,359 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:19,371 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:20,147 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 23:01:21,149 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:21,726 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:21,735 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:22,202 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 23:01:22,203 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: stop_status +2024-05-13 23:01:24,151 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:24,161 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:26,557 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:26,567 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:26,703 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:28,900 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:28,910 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:29,794 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:31,104 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:31,116 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:31,796 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:31,984 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 23:01:31,985 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 23:01:31,991 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 23:01:32,050 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 23:01:32,051 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 23:01:32,797 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav +2024-05-13 23:01:32,797 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json +2024-05-13 23:01:32,797 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav +2024-05-13 23:01:32,797 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples +2024-05-13 23:01:32,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions +2024-05-13 23:01:32,875 INFO wandb-upload_3:1257956 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmp0dy8l2wz +2024-05-13 23:01:33,304 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/3wximedk-media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json +2024-05-13 23:01:33,389 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:33,399 ERROR gpu :1257956 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 23:01:33,491 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:33,599 INFO SenderThread:1257956 [sender.py:send_request_log_artifact():1455] logged artifact run-qaoje1x9-predictionsevalstep1k - {'id': 'QXJ0aWZhY3Q6ODM0OTA1MzIx', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDMwMjM1OQ==', 'latestArtifact': None}} +2024-05-13 23:01:33,599 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,600 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json with policy now +2024-05-13 23:01:33,600 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:33,600 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 23:01:33,600 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 23:01:33,798 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:01:33,824 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/fmxfuw3j-media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav +2024-05-13 23:01:33,879 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/f9ey195a-media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav +2024-05-13 23:01:33,901 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:01:33,901 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,901 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav with policy now +2024-05-13 23:01:33,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav with policy now +2024-05-13 23:01:33,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav with policy now +2024-05-13 23:01:33,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav with policy now +2024-05-13 23:01:33,902 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,902 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav with policy now +2024-05-13 23:01:33,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,903 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav with policy now +2024-05-13 23:01:33,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: files +2024-05-13 23:01:33,903 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav with policy now +2024-05-13 23:01:33,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: telemetry +2024-05-13 23:01:33,903 DEBUG SenderThread:1257956 [sender.py:send():378] send: exit +2024-05-13 23:01:33,903 INFO SenderThread:1257956 [sender.py:send_exit():585] handling exit code: 0 +2024-05-13 23:01:33,906 INFO SenderThread:1257956 [sender.py:send_exit():587] handling runtime: 7723 +2024-05-13 23:01:33,986 INFO wandb-upload_6:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/0gk25oxx-media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav +2024-05-13 23:01:34,004 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/xm3tzz4p-media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav +2024-05-13 23:01:34,018 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/9mg7firf-media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav +2024-05-13 23:01:34,024 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/we9rf8az-media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav +2024-05-13 23:01:34,025 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/kqnjmta5-media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json +2024-05-13 23:01:34,041 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/tuqrjrmb-media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav +2024-05-13 23:01:34,477 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:01:34,477 INFO SenderThread:1257956 [sender.py:send_exit():593] send defer +2024-05-13 23:01:34,477 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:34,478 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:34,478 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 0 +2024-05-13 23:01:34,478 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:34,478 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 0 +2024-05-13 23:01:34,478 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 1 +2024-05-13 23:01:34,478 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:34,478 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 1 +2024-05-13 23:01:34,479 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:34,479 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 1 +2024-05-13 23:01:34,479 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 2 +2024-05-13 23:01:34,479 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:34,479 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 2 +2024-05-13 23:01:34,479 INFO HandlerThread:1257956 [system_monitor.py:finish():203] Stopping system monitor +2024-05-13 23:01:34,479 DEBUG SystemMonitor:1257956 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-13 23:01:34,480 DEBUG SystemMonitor:1257956 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-13 23:01:34,480 INFO HandlerThread:1257956 [interfaces.py:finish():200] Joined cpu monitor +2024-05-13 23:01:34,482 INFO HandlerThread:1257956 [interfaces.py:finish():200] Joined disk monitor +2024-05-13 23:01:34,799 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:01:34,899 INFO wandb-upload_0:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/n09f170t-media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav +2024-05-13 23:01:35,041 INFO wandb-upload_2:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/h6xp0rko-media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav +2024-05-13 23:01:35,055 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/77pqyzc6-media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav +2024-05-13 23:01:35,066 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/ekemvu9m-media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav +2024-05-13 23:01:35,148 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/bt303oft-media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav +2024-05-13 23:01:35,800 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:35,994 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/p0zwlfn5-media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav +2024-05-13 23:01:36,192 INFO wandb-upload_4:1257956 [upload_job.py:push():130] Uploaded file /tmp/tmpf4qesxn8wandb/uaw9etzp-media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav +2024-05-13 23:01:36,801 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:39,418 ERROR gpu :1257956 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 23:01:39,419 INFO HandlerThread:1257956 [interfaces.py:finish():200] Joined gpu monitor +2024-05-13 23:01:39,419 INFO HandlerThread:1257956 [interfaces.py:finish():200] Joined memory monitor +2024-05-13 23:01:39,419 INFO HandlerThread:1257956 [interfaces.py:finish():200] Joined network monitor +2024-05-13 23:01:39,419 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:39,421 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,421 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 2 +2024-05-13 23:01:39,421 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 3 +2024-05-13 23:01:39,421 DEBUG SenderThread:1257956 [sender.py:send():378] send: stats +2024-05-13 23:01:39,422 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:39,423 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:39,423 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 3 +2024-05-13 23:01:39,423 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:39,424 DEBUG SenderThread:1257956 [sender.py:send():378] send: history +2024-05-13 23:01:39,425 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: summary_record +2024-05-13 23:01:39,425 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:01:39,425 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,425 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 3 +2024-05-13 23:01:39,425 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 4 +2024-05-13 23:01:39,426 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:39,426 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 4 +2024-05-13 23:01:39,426 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,426 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 4 +2024-05-13 23:01:39,426 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 5 +2024-05-13 23:01:39,426 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:39,426 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 5 +2024-05-13 23:01:39,429 DEBUG SenderThread:1257956 [sender.py:send():378] send: summary +2024-05-13 23:01:39,429 INFO SenderThread:1257956 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 23:01:39,429 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,429 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 5 +2024-05-13 23:01:39,429 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 6 +2024-05-13 23:01:39,430 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:39,430 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 6 +2024-05-13 23:01:39,430 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,430 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 6 +2024-05-13 23:01:39,432 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:39,493 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: keepalive +2024-05-13 23:01:39,493 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:39,583 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 7 +2024-05-13 23:01:39,583 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:39,584 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:39,584 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 7 +2024-05-13 23:01:39,585 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:39,585 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 7 +2024-05-13 23:01:39,804 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/config.yaml +2024-05-13 23:01:39,804 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:01:40,494 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:41,278 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 8 +2024-05-13 23:01:41,278 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:41,279 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:41,279 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 8 +2024-05-13 23:01:41,279 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:41,279 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 8 +2024-05-13 23:01:41,279 INFO SenderThread:1257956 [job_builder.py:build():432] Attempting to build job artifact +2024-05-13 23:01:41,280 INFO SenderThread:1257956 [job_builder.py:_get_source_type():565] is repo sourced job +2024-05-13 23:01:41,282 INFO SenderThread:1257956 [job_builder.py:build():541] adding wandb-job metadata file +2024-05-13 23:01:41,289 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 9 +2024-05-13 23:01:41,289 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:41,289 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 9 +2024-05-13 23:01:41,290 DEBUG SenderThread:1257956 [sender.py:send():378] send: artifact +2024-05-13 23:01:41,494 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:41,806 INFO Thread-12 :1257956 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:42,002 INFO wandb-upload_0:1257956 [upload_job.py:push():85] Skipped uploading /home/sanchit/.local/share/wandb/artifacts/staging/tmpantujzmu +2024-05-13 23:01:42,278 INFO wandb-upload_6:1257956 [upload_job.py:push():88] Uploaded file /tmp/tmpcxnm05wb/wandb-job.json +2024-05-13 23:01:43,120 INFO SenderThread:1257956 [sender.py:send_artifact():1467] sent artifact job-https___huggingface.co_sanchit-gandhi_parler-tts-mini-v0.1-expresso-concatenated-combined_run_parler_tts_training.py - {'id': 'QXJ0aWZhY3Q6ODM0OTA1NDk0', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDIzMTI3NQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6ODM0NzI5NzMx', 'versionIndex': 0}}} +2024-05-13 23:01:43,120 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:43,120 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 9 +2024-05-13 23:01:43,120 INFO SenderThread:1257956 [dir_watcher.py:finish():358] shutting down directory watcher +2024-05-13 23:01:43,807 INFO SenderThread:1257956 [dir_watcher.py:finish():388] scan: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/conda-environment.yaml conda-environment.yaml +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json wandb-summary.json +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log output.log +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/requirements.txt requirements.txt +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/config.yaml config.yaml +2024-05-13 23:01:43,808 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-metadata.json wandb-metadata.json +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav media/audio/Speech samples/eval_254_b3650df61e399b05257d.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav media/audio/Speech samples/eval_381_7c47ba927ac118ffaacc.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav media/audio/Speech samples/eval_889_2b4fc3e5014cfe3b1528.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav media/audio/Speech samples/eval_1016_3cd94e4824cc6c8fb09c.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav media/audio/Speech samples/eval_762_09db3d4c95c46a3bae02.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav media/audio/Speech samples/eval_1016_345bfb6a72849809d361.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav media/audio/Speech samples/eval_762_b8be5a72ea66619af007.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav media/audio/Speech samples/eval_635_f5d78399080a22adb894.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav media/audio/Speech samples/eval_127_0db946e177a69cbe11f5.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav media/audio/Speech samples/eval_381_cd644667186ae0518a3c.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav media/audio/Speech samples/eval_889_b328d00c21f85f4f8675.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav media/audio/Speech samples/eval_1016_9da1fed11be9d614d9ec.wav +2024-05-13 23:01:43,816 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav media/audio/Speech samples/eval_127_43ed5d3749c912acb591.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav media/audio/Speech samples/eval_1016_ec838b0233dbe87d33f3.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav media/audio/Speech samples/eval_635_42a65927176944bbb79c.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav media/audio/Speech samples/eval_635_0eae5cc1510f147a4816.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav media/audio/Speech samples/eval_127_d24330f3382b9e6ea7ea.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav media/audio/Speech samples/eval_889_a6fbe90ed6a065a3951a.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav media/audio/Speech samples/eval_508_5619d97860f92fc1a62d.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav media/audio/Speech samples/eval_508_fe4bb0e429ec01f84b9e.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav media/audio/Speech samples/eval_254_2794dcaf322bd12e2814.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav media/audio/Speech samples/eval_762_7ab4dededd2d3a052def.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav media/audio/Speech samples/eval_127_32c9af8d48e757598000.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav media/audio/Speech samples/eval_635_57893f23cc6a3298ca26.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav media/audio/Speech samples/eval_635_60380fa0c3bbae748090.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav media/audio/Speech samples/eval_1016_3c9adbd9374e0fb5ce3d.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_492597073098578f0605.wav media/audio/Speech samples/eval_381_492597073098578f0605.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav media/audio/Speech samples/eval_508_bcb03b95f0470920bdc6.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav media/audio/Speech samples/eval_254_4ca836a112634417b82e.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav media/audio/Speech samples/eval_508_00e9064c0bdbd6b9428d.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav media/audio/Speech samples/eval_889_f4a80b0f8d211d8b3017.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav media/audio/Speech samples/eval_381_67d9409a306e3614ec3f.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav media/audio/Speech samples/eval_508_e48b4ff2b12d5ffdb11c.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav media/audio/Speech samples/eval_254_837a3499e3f93538b643.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav media/audio/Speech samples/eval_1016_193b029d494fd24e7cfa.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav media/audio/Speech samples/eval_127_ec7dcb5421538131ede7.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav media/audio/Speech samples/eval_889_19eb66d9eec8cfe9f4bd.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav media/audio/Speech samples/eval_381_89ad32d31f3e70178cc1.wav +2024-05-13 23:01:43,817 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav media/audio/Speech samples/eval_127_75818e76e9e077f058be.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav media/audio/Speech samples/eval_762_3349ee3cb71e3d789f00.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav media/audio/Speech samples/eval_1016_3ed6544e58dd861a5d9e.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav media/audio/Speech samples/eval_254_e8ca5038019cad3cde86.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav media/audio/Speech samples/eval_127_341c52fd92336c009f67.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav media/audio/Speech samples/eval_635_a642b404a0e07a7f2886.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav media/audio/Speech samples/eval_381_f7405ef7b645b3265477.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav media/audio/Speech samples/eval_508_edb524db4e5408a9c0f1.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav media/audio/Speech samples/eval_381_4899e0da4615e883ad13.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav media/audio/Speech samples/eval_254_d33ccbefe990db0dce2b.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav media/audio/Speech samples/eval_254_2ef5b33e2eaf98dca4a6.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav media/audio/Speech samples/eval_762_63964662df5bc7731b7a.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav media/audio/Speech samples/eval_762_e8feadb7228a5b136ce8.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav media/audio/Speech samples/eval_508_11adac906cb7e2ef30c6.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_4871014c416be45744ef.wav media/audio/Speech samples/eval_889_4871014c416be45744ef.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav media/audio/Speech samples/eval_635_1643864ecc4e4e44c2c6.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_889_819325490f684d367b39.wav media/audio/Speech samples/eval_889_819325490f684d367b39.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav media/audio/Speech samples/eval_762_d1c21fd700287f9f3109.wav +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json media/table/predictions/eval-step-635_635_8b369ee796f15026de8f.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json media/table/predictions/eval-step-508_505_ebd8a6e2299806a596d7.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json media/table/predictions/eval-step-381_380_1bf220a613a57386890a.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json media/table/predictions/eval-step-254_250_eb7b50d0179e2d66397e.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json media/table/predictions/eval-step-889_885_9d5bd9e7754ccad0a997.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json media/table/predictions/eval-step-127_125_a3f59299480689ee898f.table.json +2024-05-13 23:01:43,818 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json media/table/predictions/eval-step-762_760_df928dba2c214ae9137f.table.json +2024-05-13 23:01:43,819 INFO SenderThread:1257956 [dir_watcher.py:finish():402] scan save: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json media/table/predictions/eval-step-1k_1015_6ea072f5a546055d8795.table.json +2024-05-13 23:01:43,819 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 10 +2024-05-13 23:01:43,819 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:43,819 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:43,819 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 10 +2024-05-13 23:01:43,820 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:43,820 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 10 +2024-05-13 23:01:43,820 INFO SenderThread:1257956 [file_pusher.py:finish():169] shutting down file pusher +2024-05-13 23:01:44,141 INFO wandb-upload_5:1257956 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/conda-environment.yaml +2024-05-13 23:01:44,146 INFO wandb-upload_8:1257956 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/requirements.txt +2024-05-13 23:01:44,153 INFO wandb-upload_3:1257956 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/config.yaml +2024-05-13 23:01:44,161 INFO wandb-upload_7:1257956 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/wandb-summary.json +2024-05-13 23:01:44,374 INFO wandb-upload_1:1257956 [upload_job.py:push():130] Uploaded file /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/files/output.log +2024-05-13 23:01:44,496 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:44,496 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:44,497 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 23:01:44,575 INFO Thread-11 (_thread_body):1257956 [sender.py:transition_state():613] send defer: 11 +2024-05-13 23:01:44,575 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:44,575 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 11 +2024-05-13 23:01:44,576 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:44,576 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 11 +2024-05-13 23:01:44,576 INFO SenderThread:1257956 [file_pusher.py:join():175] waiting for file pusher +2024-05-13 23:01:44,584 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 12 +2024-05-13 23:01:44,584 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:44,584 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 12 +2024-05-13 23:01:44,585 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:44,585 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 12 +2024-05-13 23:01:44,585 INFO SenderThread:1257956 [file_stream.py:finish():601] file stream finish called +2024-05-13 23:01:45,112 INFO SenderThread:1257956 [file_stream.py:finish():605] file stream finish is done +2024-05-13 23:01:45,112 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 13 +2024-05-13 23:01:45,112 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:45,112 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 13 +2024-05-13 23:01:45,113 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:45,113 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 13 +2024-05-13 23:01:45,113 INFO SenderThread:1257956 [sender.py:transition_state():613] send defer: 14 +2024-05-13 23:01:45,113 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: defer +2024-05-13 23:01:45,113 INFO HandlerThread:1257956 [handler.py:handle_request_defer():184] handle defer: 14 +2024-05-13 23:01:45,114 DEBUG SenderThread:1257956 [sender.py:send():378] send: final +2024-05-13 23:01:45,114 DEBUG SenderThread:1257956 [sender.py:send():378] send: footer +2024-05-13 23:01:45,114 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: defer +2024-05-13 23:01:45,114 INFO SenderThread:1257956 [sender.py:send_request_defer():609] handle sender defer: 14 +2024-05-13 23:01:45,115 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: poll_exit +2024-05-13 23:01:45,115 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: poll_exit +2024-05-13 23:01:45,116 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: internal_messages +2024-05-13 23:01:45,117 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: server_info +2024-05-13 23:01:45,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: get_summary +2024-05-13 23:01:45,118 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: sampled_history +2024-05-13 23:01:45,120 DEBUG SenderThread:1257956 [sender.py:send_request():405] send_request: server_info +2024-05-13 23:01:45,251 DEBUG HandlerThread:1257956 [handler.py:handle_request():158] handle_request: shutdown +2024-05-13 23:01:45,252 INFO HandlerThread:1257956 [handler.py:finish():882] shutting down handler +2024-05-13 23:01:46,118 INFO WriterThread:1257956 [datastore.py:close():296] close: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/run-qaoje1x9.wandb +2024-05-13 23:01:46,250 INFO SenderThread:1257956 [sender.py:finish():1545] shutting down sender +2024-05-13 23:01:46,250 INFO SenderThread:1257956 [file_pusher.py:finish():169] shutting down file pusher +2024-05-13 23:01:46,250 INFO SenderThread:1257956 [file_pusher.py:join():175] waiting for file pusher diff --git a/wandb/run-20240513_205249-qaoje1x9/logs/debug.log b/wandb/run-20240513_205249-qaoje1x9/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..eec1410311289077b92c6ea54355d9093f5a6ed4 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/logs/debug.log @@ -0,0 +1,35 @@ +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Current SDK version is 0.17.0 +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Configure stats pid to 1257680 +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Loading settings from /home/sanchit/.config/wandb/settings +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Loading settings from /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/settings +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'run_parler_tts_training.py', 'program_abspath': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py', 'program': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/run_parler_tts_training.py'} +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:_log_setup():520] Logging user logs to /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/logs/debug.log +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:_log_setup():521] Logging internal logs to /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated-combined/wandb/run-20240513_205249-qaoje1x9/logs/debug-internal.log +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:init():560] calling init triggers +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:init():567] wandb.init called with sweep_config: {} +config: {} +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:init():610] starting backend +2024-05-13 20:52:49,015 INFO MainThread:1257680 [wandb_init.py:init():614] setting up manager +2024-05-13 20:52:49,019 INFO MainThread:1257680 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-13 20:52:49,019 INFO MainThread:1257680 [wandb_init.py:init():622] backend started and connected +2024-05-13 20:52:49,021 INFO MainThread:1257680 [wandb_init.py:init():711] updated telemetry +2024-05-13 20:52:49,024 INFO MainThread:1257680 [wandb_init.py:init():744] communicating run to backend with 90.0 second timeout +2024-05-13 20:52:49,413 INFO MainThread:1257680 [wandb_run.py:_on_init():2396] communicating current version +2024-05-13 20:52:49,474 INFO MainThread:1257680 [wandb_run.py:_on_init():2405] got version response +2024-05-13 20:52:49,475 INFO MainThread:1257680 [wandb_init.py:init():795] starting run threads in backend +2024-05-13 20:52:52,103 INFO MainThread:1257680 [wandb_run.py:_console_start():2374] atexit reg +2024-05-13 20:52:52,103 INFO MainThread:1257680 [wandb_run.py:_redirect():2229] redirect: wrap_raw +2024-05-13 20:52:52,103 INFO MainThread:1257680 [wandb_run.py:_redirect():2294] Wrapping output streams. +2024-05-13 20:52:52,103 INFO MainThread:1257680 [wandb_run.py:_redirect():2319] Redirects installed. +2024-05-13 20:52:52,104 INFO MainThread:1257680 [wandb_init.py:init():838] run started, returning control to user process +2024-05-13 20:52:52,104 INFO MainThread:1257680 [wandb_run.py:_config_callback():1376] config_cb None None {'learning_rate': 8e-05, 'model_name_or_path': 'parler-tts/parler_tts_mini_v0.1', 'num_train_epochs': 8, 'gradient_accumulation_steps': 8, 'per_device_train_batch_size': 16, 'global_batch_size': 16, 'mixed_precision': 'bf16', 'lr_scheduler_type': 'SchedulerType.COSINE', 'warmup_steps': 250, 'freeze_text_encoder': True, 'max_duration_in_seconds': 30.0, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.99, 'temperature': 1.0} +2024-05-13 23:01:32,489 INFO MainThread:1257680 [wandb_run.py:_finish():2103] finishing run sanchit-gandhi/parler-speech/qaoje1x9 +2024-05-13 23:01:32,489 INFO MainThread:1257680 [wandb_run.py:_atexit_cleanup():2343] got exitcode: 0 +2024-05-13 23:01:32,489 INFO MainThread:1257680 [wandb_run.py:_restore():2326] restore +2024-05-13 23:01:32,489 INFO MainThread:1257680 [wandb_run.py:_restore():2332] restore done +2024-05-13 23:01:46,253 INFO MainThread:1257680 [wandb_run.py:_footer_history_summary_info():3994] rendering history +2024-05-13 23:01:46,254 INFO MainThread:1257680 [wandb_run.py:_footer_history_summary_info():4026] rendering summary +2024-05-13 23:01:46,256 INFO MainThread:1257680 [wandb_run.py:_footer_sync_info():3953] logging synced files diff --git a/wandb/run-20240513_205249-qaoje1x9/run-qaoje1x9.wandb b/wandb/run-20240513_205249-qaoje1x9/run-qaoje1x9.wandb new file mode 100644 index 0000000000000000000000000000000000000000..472b8834281450d500f315d6fd86a56fc31d1ec9 --- /dev/null +++ b/wandb/run-20240513_205249-qaoje1x9/run-qaoje1x9.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c82c2f3fd25babb384593c79e20521a350a44da83ba94832de9b91c6ce06e8d +size 2892622