Delete egs/ami/ASR/.ipynb_checkpoints

egs/ami/ASR/.ipynb_checkpoints/prepare-checkpoint.sh

@@ -1,108 +0,0 @@
-#!/usr/bin/env bash
-
-set -eou pipefail
-
-stage=-1
-stop_stage=100
-use_gss=false   # ❌ Disable GSS (not needed for IHM-only)
-
-dl_dir=$PWD/download
-
-. shared/parse_options.sh || exit 1
-
-mkdir -p data
-vocab_size=500
-
-log() {
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-log "dl_dir: $dl_dir"
-
-# -----------------------------
-# Stage 0: Download AMI (IHM only)
-# -----------------------------
-if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then
-  log "Stage 0: Download AMI IHM data (~70h)"
-
-  if [ ! -d $dl_dir/amicorpus ]; then
-    lhotse download ami --mic ihm $dl_dir/amicorpus
-  fi
-fi
-
-# -----------------------------
-# Stage 1: Prepare AMI IHM manifests
-# -----------------------------
-if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
-  log "Stage 1: Prepare AMI IHM manifests"
-
-  mkdir -p data/manifests
-
-  lhotse prepare ami \
-    --mic ihm \
-    --partition full-corpus-asr \
-    --normalize-text kaldi \
-    --max-words-per-segment 30 \
-    $dl_dir/amicorpus data/manifests/
-fi
-
-# -----------------------------
-# Stage 4: Compute fbank features (AMI IHM only)
-# -----------------------------
-if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then
-  log "Stage 4: Compute fbank features for AMI IHM"
-
-  mkdir -p data/fbank
-  python local/compute_fbank_ami.py
-
-  log "Creating final IHM training cuts"
-  gunzip -c data/manifests/cuts_train_ihm.jsonl.gz | \
-    gzip -c > data/manifests/cuts_train_all.jsonl.gz
-fi
-
-# -----------------------------
-# Stage 6: Dump transcripts for BPE training (IHM only)
-# -----------------------------
-if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then
-  log "Stage 6: Dump IHM transcripts for BPE"
-
-  mkdir -p data/lm
-
-  gunzip -c data/manifests/ami-ihm_supervisions_train.jsonl.gz | \
-    jq '.text' | sed 's:"::g' > data/lm/transcript_words.txt
-fi
-
-# -----------------------------
-# Stage 7: Prepare BPE-based language resources
-# -----------------------------
-if [ $stage -le 7 ] && [ $stop_stage -ge 7 ]; then
-  log "Stage 7: Prepare BPE-based lang"
-
-  lang_dir=data/lang_bpe_${vocab_size}
-  mkdir -p $lang_dir
-
-  echo "<eps> 0" > $lang_dir/words.txt
-  echo "!SIL 1" >> $lang_dir/words.txt
-  echo "<UNK> 2" >> $lang_dir/words.txt
-
-  cat data/lm/transcript_words.txt | \
-    grep -o -E '\w+' | sort -u | \
-    awk '{print $0, NR+2}' >> $lang_dir/words.txt
-
-  num_words=$(wc -l < $lang_dir/words.txt)
-  echo "<s> ${num_words}" >> $lang_dir/words.txt
-  num_words=$(wc -l < $lang_dir/words.txt)
-  echo "</s> ${num_words}" >> $lang_dir/words.txt
-  num_words=$(wc -l < $lang_dir/words.txt)
-  echo "#0 ${num_words}" >> $lang_dir/words.txt
-
-  ./local/train_bpe_model.py \
-    --lang-dir $lang_dir \
-    --vocab-size $vocab_size \
-    --transcript data/lm/transcript_words.txt
-
-  if [ ! -f $lang_dir/L_disambig.pt ]; then
-    ./local/prepare_lang_bpe.py --lang-dir $lang_dir
-  fi
-fi

egs/ami/ASR/.ipynb_checkpoints/prepare_scd-checkpoint.sh

@@ -1,274 +0,0 @@
-#!/usr/bin/env bash
-# Modified prepare.sh for AMI with [SCD] token augmentation
-# This script adds Speaker Change Detection tokens to the data
-
-set -eou pipefail
-
-stage=2  # Skip download (stage 0) and manifest prep (stage 1) - use existing data
-stop_stage=100
-use_gss=false   # ❌ Disable GSS (not needed for IHM-only)
-
-dl_dir=$PWD/download
-
-. shared/parse_options.sh || exit 1
-
-mkdir -p data
-vocab_size=500
-
-log() {
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-log "dl_dir: $dl_dir"
-
-# -----------------------------
-# Stage 0: Download AMI (IHM only)
-# -----------------------------
-if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then
-  log "Stage 0: Download AMI IHM data (~70h)"
-
-  if [ ! -d $dl_dir/amicorpus ]; then
-    lhotse download ami --mic ihm $dl_dir/amicorpus
-  fi
-fi
-
-# -----------------------------
-# Stage 1: Prepare AMI IHM manifests
-# -----------------------------
-if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
-  log "Stage 1: Prepare AMI IHM manifests"
-
-  mkdir -p data/manifests
-
-  lhotse prepare ami \
-    --mic ihm \
-    --partition full-corpus-asr \
-    --normalize-text kaldi \
-    --max-words-per-segment 30 \
-    $dl_dir/amicorpus data/manifests/
-fi
-
-# -----------------------------
-# Stage 2: Add [SCD] tokens (Speaker Change Detection)
-# -----------------------------
-if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
-  log "Stage 2: Merge utterances and add [SCD] tokens"
-
-  mkdir -p data/manifests_scd
-
-  # Process train set
-  log "Adding [SCD] tokens to train set"
-  python local/add_scd_tokens.py \
-    --input-manifest data/manifests/ami-ihm_supervisions_train.jsonl.gz \
-    --output-manifest data/manifests_scd/ami-ihm_supervisions_train_scd.jsonl.gz \
-    --max-duration 20.0 \
-    --min-duration 5.0 \
-    --gap-threshold 2.0
-
-  # Process dev set
-  log "Adding [SCD] tokens to dev set"
-  python local/add_scd_tokens.py \
-    --input-manifest data/manifests/ami-ihm_supervisions_dev.jsonl.gz \
-    --output-manifest data/manifests_scd/ami-ihm_supervisions_dev_scd.jsonl.gz \
-    --max-duration 20.0 \
-    --min-duration 5.0 \
-    --gap-threshold 2.0
-
-  # Process test set
-  log "Adding [SCD] tokens to test set"
-  python local/add_scd_tokens.py \
-    --input-manifest data/manifests/ami-ihm_supervisions_test.jsonl.gz \
-    --output-manifest data/manifests_scd/ami-ihm_supervisions_test_scd.jsonl.gz \
-    --max-duration 20.0 \
-    --min-duration 5.0 \
-    --gap-threshold 2.0
-
-  log "Created [SCD]-augmented supervisions in data/manifests_scd/"
-fi
-
-# -----------------------------
-# Stage 3: Create cuts with [SCD] supervisions
-# -----------------------------
-if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
-  log "Stage 3: Create cuts using [SCD]-augmented supervisions"
-
-  mkdir -p data/cuts_scd
-
-  # For each split, combine recordings with [SCD]-augmented supervisions
-  for split in train dev test; do
-    log "Processing $split split with [SCD] supervisions"
-
-    # Use original recordings but [SCD]-augmented supervisions
-    python -c "
-from lhotse import load_manifest, CutSet
-
-recordings = load_manifest('data/manifests/ami-ihm_recordings_${split}.jsonl.gz')
-supervisions = load_manifest('data/manifests_scd/ami-ihm_supervisions_${split}_scd.jsonl.gz')
-
-cuts = CutSet.from_manifests(
-    recordings=recordings,
-    supervisions=supervisions
-).trim_to_supervisions(keep_overlapping=False, keep_all_channels=False)
-
-# Use Lhotse's built-in serialization
-cuts.to_file('data/cuts_scd/cuts_${split}_ihm_scd.jsonl.gz')
-
-print(f'Created cuts for ${split}')
-"
-  done
-
-  log "Created cuts with [SCD] tokens in data/cuts_scd/"
-fi
-
-# -----------------------------
-# Stage 4: Compute fbank features for [SCD] cuts
-# -----------------------------
-if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then
-  log "Stage 4: Compute fbank features for [SCD]-augmented cuts"
-
-  mkdir -p data/fbank_scd
-
-  # Process train cuts with speed perturbation
-  log "Processing train cuts with [SCD] tokens"
-  python -c "
-import torch
-from lhotse import CutSet, LilcomChunkyWriter
-from lhotse.features.kaldifeat import (
-    KaldifeatFbank,
-    KaldifeatFbankConfig,
-    KaldifeatFrameOptions,
-    KaldifeatMelOptions,
-)
-from pathlib import Path
-
-torch.set_num_threads(1)
-torch.set_num_interop_threads(1)
-
-extractor = KaldifeatFbank(
-    KaldifeatFbankConfig(
-        frame_opts=KaldifeatFrameOptions(sampling_rate=16000),
-        mel_opts=KaldifeatMelOptions(num_bins=80),
-        device='cuda',
-    )
-)
-
-cuts = CutSet.from_file('data/cuts_scd/cuts_train_ihm_scd.jsonl.gz')
-cuts = cuts + cuts.perturb_speed(0.9) + cuts.perturb_speed(1.1)
-
-cuts.compute_and_store_features_batch(
-    extractor=extractor,
-    storage_path=Path('data/fbank_scd/feats_train_ihm_scd'),
-    manifest_path=Path('data/manifests_scd/cuts_train_ihm_scd.jsonl.gz'),
-    batch_duration=5000,
-    num_workers=8,
-    storage_type=LilcomChunkyWriter,
-)
-print('Train features computed')
-"
-
-  # Process dev and test (no speed perturbation)
-  for split in dev test; do
-    log "Processing $split cuts with [SCD] tokens"
-    python -c "
-import torch
-from lhotse import CutSet, LilcomChunkyWriter
-from lhotse.features.kaldifeat import (
-    KaldifeatFbank,
-    KaldifeatFbankConfig,
-    KaldifeatFrameOptions,
-    KaldifeatMelOptions,
-)
-from pathlib import Path
-
-torch.set_num_threads(1)
-torch.set_num_interop_threads(1)
-
-extractor = KaldifeatFbank(
-    KaldifeatFbankConfig(
-        frame_opts=KaldifeatFrameOptions(sampling_rate=16000),
-        mel_opts=KaldifeatMelOptions(num_bins=80),
-        device='cuda',
-    )
-)
-
-cuts = CutSet.from_file('data/cuts_scd/cuts_${split}_ihm_scd.jsonl.gz')
-
-cuts.compute_and_store_features_batch(
-    extractor=extractor,
-    storage_path=Path('data/fbank_scd/feats_${split}_ihm_scd'),
-    manifest_path=Path('data/manifests_scd/cuts_${split}_ihm_scd.jsonl.gz'),
-    batch_duration=5000,
-    num_workers=4,
-    storage_type=LilcomChunkyWriter,
-)
-print('${split} features computed')
-"
-  done
-
-  log "Features with [SCD] tokens computed"
-fi
-
-# -----------------------------
-# Stage 5: Dump transcripts with [SCD] tokens for BPE training
-# -----------------------------
-if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
-  log "Stage 5: Dump transcripts with [SCD] tokens for BPE"
-
-  mkdir -p data/lm_scd
-
-  gunzip -c data/manifests_scd/ami-ihm_supervisions_train_scd.jsonl.gz | \
-    jq '.text' | sed 's:"::g' > data/lm_scd/transcript_words_scd.txt
-
-  log "Transcripts with [SCD] tokens saved to data/lm_scd/transcript_words_scd.txt"
-
-  # Show sample with [SCD] tokens
-  log "Sample transcripts with [SCD] tokens:"
-  head -5 data/lm_scd/transcript_words_scd.txt
-fi
-
-# -----------------------------
-# Stage 6: Train BPE model with [SCD] as single token
-# -----------------------------
-if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then
-  log "Stage 6: Train BPE model with [SCD] token"
-
-  lang_dir=data/lang_bpe_${vocab_size}_scd
-  mkdir -p $lang_dir
-
-  echo "<eps> 0" > $lang_dir/words.txt
-  echo "!SIL 1" >> $lang_dir/words.txt
-  echo "<UNK> 2" >> $lang_dir/words.txt
-
-  cat data/lm_scd/transcript_words_scd.txt | \
-    grep -o -E '\w+|\[SCD\]' | sort -u | \
-    awk '{print $0, NR+2}' >> $lang_dir/words.txt
-
-  num_words=$(wc -l < $lang_dir/words.txt)
-  echo "<s> ${num_words}" >> $lang_dir/words.txt
-  num_words=$(wc -l < $lang_dir/words.txt)
-  echo "</s> ${num_words}" >> $lang_dir/words.txt
-  num_words=$(wc -l < $lang_dir/words.txt)
-  echo "#0 ${num_words}" >> $lang_dir/words.txt
-
-  # Use modified BPE trainer that treats [SCD] as single token
-  ./local/train_bpe_model_scd.py \
-    --lang-dir $lang_dir \
-    --vocab-size $vocab_size \
-    --transcript data/lm_scd/transcript_words_scd.txt \
-    --task-tokens "[SCD]"
-
-  if [ ! -f $lang_dir/L_disambig.pt ]; then
-    ./local/prepare_lang_bpe.py --lang-dir $lang_dir
-  fi
-
-  log "BPE model with [SCD] token trained successfully"
-fi
-
-log "Data preparation with [SCD] tokens complete!"
-log ""
-log "Next steps:"
-log "1. Verify [SCD] tokens in: data/lm_scd/transcript_words_scd.txt"
-log "2. Check BPE model includes [SCD] as single token: data/lang_bpe_${vocab_size}_scd/tokens.txt"
-log "3. Use data/manifests_scd/cuts_*_ihm_scd.jsonl.gz for training"
-log "4. Update your training script to use lang_dir=data/lang_bpe_${vocab_size}_scd"