| import tensorflow as tf
|
| from tensorflow.keras.preprocessing.sequence import pad_sequences
|
| from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional, Dropout
|
| from tensorflow.keras.preprocessing.text import Tokenizer
|
| from tensorflow.keras.models import Sequential
|
| from tensorflow.keras.optimizers import Adam
|
| from tensorflow.keras.callbacks import EarlyStopping
|
| import numpy as np
|
| import re
|
|
|
|
|
|
|
|
|
| def preprocess_line(line):
|
| line = re.sub(r"<\/?s>", "", line)
|
| line = re.sub(r"\(\d+\)", "", line)
|
| line = line.replace("_", "")
|
| line = line.replace("7", "h")
|
| return line.strip()
|
|
|
|
|
|
|
|
|
| with open("Phones.txt", encoding="utf-8") as f:
|
| phones = [p.strip() for p in f if p.strip()]
|
| print(f"Loaded {len(phones)} phones from inventory")
|
|
|
|
|
|
|
|
|
| with open("Transcription-ROMAN.txt", encoding="utf-8") as f:
|
| raw_lines = [preprocess_line(line) for line in f if line.strip()]
|
|
|
|
|
| corpus = []
|
| for line in raw_lines:
|
| tokens = line.split()
|
| if tokens:
|
| corpus.append(" ".join(tokens))
|
|
|
| print(f"Corpus size: {len(corpus)} lines")
|
| print("Example preprocessed line:", corpus[0][:200])
|
|
|
|
|
|
|
|
|
| tokenizer = Tokenizer(num_words=10000, oov_token="<OOV>")
|
|
|
| tokenizer.fit_on_texts(corpus)
|
|
|
| total_vocab = len(tokenizer.word_index) + 1
|
| print("Total vocab size:", total_vocab)
|
|
|
|
|
|
|
|
|
| input_sequences = []
|
| for line in corpus:
|
| token_list = tokenizer.texts_to_sequences([line])[0]
|
| for i in range(1, len(token_list)):
|
| n_gram_seq = token_list[:i+1]
|
| input_sequences.append(n_gram_seq)
|
|
|
| max_sequence_len = max(len(x) for x in input_sequences)
|
| print("Max sequence length:", max_sequence_len)
|
|
|
| input_sequences = np.array(
|
| pad_sequences(input_sequences, maxlen=max_sequence_len, padding="pre")
|
| )
|
|
|
| xs, labels = input_sequences[:, :-1], input_sequences[:, -1]
|
|
|
|
|
|
|
|
|
| model = Sequential([
|
| Embedding(total_vocab, 128),
|
| Bidirectional(LSTM(256, return_sequences=True)),
|
| Dropout(0.3),
|
| LSTM(128),
|
| Dense(256, activation="relu"),
|
| Dropout(0.3),
|
| Dense(total_vocab, activation="softmax")
|
| ])
|
|
|
| adam = Adam(learning_rate=0.0005)
|
| model.compile(loss="sparse_categorical_crossentropy",
|
| optimizer=adam,
|
| metrics=["accuracy"])
|
|
|
|
|
| model.build(input_shape=(None, max_sequence_len-1))
|
| print(model.summary())
|
|
|
|
|
|
|
|
|
| early_stop = EarlyStopping(monitor="val_loss", patience=7, restore_best_weights=True)
|
|
|
| history = model.fit(
|
| xs, labels,
|
| epochs=50,
|
| batch_size=128,
|
| validation_split=0.1,
|
| shuffle=True,
|
| callbacks=[early_stop],
|
| verbose=1
|
| )
|
|
|
|
|
|
|
|
|
| def sample_with_temp(preds, temperature=1.0):
|
| preds = np.asarray(preds).astype("float64")
|
| preds = np.log(preds + 1e-8) / temperature
|
| exp_preds = np.exp(preds)
|
| preds = exp_preds / np.sum(exp_preds)
|
| return np.random.choice(len(preds), p=preds)
|
|
|
| def generate_sequence(seed_text, next_tokens=30, temperature=0.8):
|
| """Generate a phoneme sequence from a seed"""
|
| for _ in range(next_tokens):
|
| token_list = tokenizer.texts_to_sequences([seed_text])[0]
|
| token_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding="pre")
|
| preds = model.predict(token_list, verbose=0)[0]
|
| predicted = sample_with_temp(preds, temperature)
|
| output_token = tokenizer.index_word.get(predicted, "")
|
| seed_text += " " + output_token
|
| return seed_text
|
|
|
| def reconstruct_words(sequence: str) -> str:
|
| """Join phoneme tokens back into Roman Urdu words"""
|
| tokens = sequence.split()
|
| return " ".join(tokens).replace(" ", " ").strip()
|
|
|
|
|
|
|
|
|
| seed = "zindagi ek"
|
| raw_output = generate_sequence(seed, next_tokens=7, temperature=0.7)
|
| print("Generated phoneme sequence:", raw_output)
|
|
|
| roman_urdu = reconstruct_words(raw_output)
|
| print("Reconstructed Roman Urdu:", roman_urdu)
|
|
|
