models.py

# coding:utf-8

import os
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.utils import spectral_norm
from torch.nn.utils.parametrizations import weight_norm
# from Utils.ASR.models import ASRCNN
# from Utils.JDC.model import JDCNet
from Modules.hifigan import _tile, AdainResBlk1d
import math

class MelSpec(torch.nn.Module):

    def __init__(self,
                 sample_rate=17402, # https://github.com/fakerybakery/styletts2-cli/blob/main/msinference.py = Default 16000. However 17400 vocalises better also "en_US/vctk_p274"
                 n_fft=2048,
                 win_length=1200,
                 hop_length=300,
                 n_mels=80
                 ):
        '''avoids dependency on torchaudio'''
        super().__init__()
        self.n_fft = n_fft
        self.win_length = win_length if win_length is not None else n_fft
        self.hop_length = hop_length if hop_length is not None else self.win_length // 2
        # --
        f_min = 0.0
        f_max = float(sample_rate // 2)
        all_freqs = torch.linspace(0, sample_rate // 2, n_fft//2+1)
        m_min = 2595.0 * math.log10(1.0 + (f_min / 700.0))
        m_max = 2595.0 * math.log10(1.0 + (f_max / 700.0))
        m_pts = torch.linspace(m_min, m_max, n_mels + 2)
        f_pts = 700.0 * (10 ** (m_pts / 2595.0) - 1.0)
        f_diff = f_pts[1:] - f_pts[:-1]  # (n_mels + 1)
        slopes = f_pts.unsqueeze(0) - all_freqs.unsqueeze(1)
        zero = torch.zeros(1)
        down_slopes = (-1.0 * slopes[:, :-2]) / f_diff[:-1]  # (n_freqs, n_mels)
        up_slopes = slopes[:, 2:] / f_diff[1:]  # (n_freqs, n_mels)
        fb = torch.max(zero, torch.min(down_slopes, up_slopes))
        # --
        self.register_buffer('fb', fb)
        window = torch.hann_window(self.win_length)
        self.register_buffer('window', window)

    def forward(self, x):
        spec_f = torch.stft(x,
                            self.n_fft,
                            self.hop_length,
                            self.win_length,
                            self.window,
                            center=True,
                            pad_mode="reflect",
                            normalized=False,
                            onesided=True,
                            return_complex=True)  # [bs, 1025, 56]
        mel_specgram = torch.matmul(spec_f.abs().pow(2).transpose(1, 2), self.fb).transpose(1, 2)
        return mel_specgram[:, None, :, :]  # [bs, 1, 80, time]


class LearnedDownSample(nn.Module):
    def __init__(self, dim_in):
        super().__init__()
        self.conv = spectral_norm(nn.Conv2d(dim_in, dim_in, kernel_size=(
                3, 3), stride=(2, 2), groups=dim_in, padding=1))
        
    def forward(self, x):
        return self.conv(x)


class ResBlk(nn.Module):
    def __init__(self, 
                 dim_in, dim_out):
        super().__init__()
        self.actv = nn.LeakyReLU(0.2)   # .07 also nice
        self.downsample_res = LearnedDownSample(dim_in)
        self.learned_sc = dim_in != dim_out
        self.conv1 = spectral_norm(nn.Conv2d(dim_in, dim_in, 3, 1, 1))
        self.conv2 = spectral_norm(nn.Conv2d(dim_in, dim_out, 3, 1, 1))
        if self.learned_sc:
            self.conv1x1 = spectral_norm(
                nn.Conv2d(dim_in, dim_out, 1, 1, 0, bias=False))

    def _shortcut(self, x):
        if self.learned_sc:
            x = self.conv1x1(x)
        if x.shape[3] % 2 != 0:  # [bs, 128, Freq, Time]
            x = torch.cat([x, x[:, :, :, -1:]], dim=3)
        return F.interpolate(x, scale_factor=.5, mode='nearest-exact')  # F.avg_pool2d(x, 2)

    def _residual(self, x):
        x = self.actv(x)
        x = self.conv1(x)
        x = self.downsample_res(x)
        x = self.actv(x)
        x = self.conv2(x)
        return x

    def forward(self, x):
        x = self._shortcut(x) + self._residual(x)
        return x / math.sqrt(2)  # unit variance


class StyleEncoder(nn.Module):

    #  for both acoustic & prosodic ref_s/p

    def __init__(self,
                 dim_in=64,
                 style_dim=128,
                 max_conv_dim=512):
        super().__init__()
        blocks = [spectral_norm(nn.Conv2d(1, dim_in, 3, stride=1, padding=1))]
        for _ in range(4):
            dim_out = min(dim_in * 2, 
                          max_conv_dim)
            blocks += [ResBlk(dim_in, dim_out)]
            dim_in = dim_out
        blocks += [nn.LeakyReLU(0.24),  # w/o this activation - produces no speech
                   spectral_norm(nn.Conv2d(dim_out, dim_out, 5, stride=1, padding=0)),
                   nn.LeakyReLU(0.2)  # 0.3 sounds nice
                   ]
        self.shared = nn.Sequential(*blocks)
        self.unshared = nn.Linear(dim_out, style_dim)

    def forward(self, x):
        x = self.shared(x)
        x = x.mean(3, keepdims=True)  # comment this line for time varying style vector
        x = x.transpose(1, 3)
        s = self.unshared(x)
        return s


class LinearNorm(torch.nn.Module):
    def __init__(self, in_dim, out_dim, bias=True):
        super().__init__()
        self.linear_layer = torch.nn.Linear(in_dim, out_dim, bias=bias)

    def forward(self, x):
        return self.linear_layer(x)


class LayerNorm(nn.Module):
    def __init__(self, channels, eps=1e-5):
        super().__init__()
        self.channels = channels
        self.eps = eps

        self.gamma = nn.Parameter(torch.ones(channels))
        self.beta = nn.Parameter(torch.zeros(channels))

    def forward(self, x):
        x = x.transpose(1, -1)
        x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
        return x.transpose(1, -1)


class TextEncoder(nn.Module):
    def __init__(self, channels, kernel_size, depth, n_symbols):
        super().__init__()
        self.embedding = nn.Embedding(n_symbols, channels)
        padding = (kernel_size - 1) // 2
        self.cnn = nn.ModuleList()
        for _ in range(depth):
            self.cnn.append(nn.Sequential(
                weight_norm(nn.Conv1d(channels, channels, kernel_size=kernel_size, padding=padding)),
                LayerNorm(channels),
                nn.LeakyReLU(0.24))
                            )
        self.lstm = nn.LSTM(channels, channels//2, 1,
                            batch_first=True, bidirectional=True)

    def forward(self, x):
        x = self.embedding(x)  # [B, T, emb]
        x = x.transpose(1, 2)
        for c in self.cnn:
            x = c(x)
        x = x.transpose(1, 2)
        x, _ = self.lstm(x)
        return x


class AdaLayerNorm(nn.Module):

    def __init__(self, style_dim, channels=None, eps=1e-5):
        super().__init__()
        self.eps = eps
        self.fc = nn.Linear(style_dim, 1024)

    def forward(self, x, s):
        h = self.fc(s)
        gamma = h[:, :, :512]
        beta = h[:, :, 512:1024]
        x = F.layer_norm(x, (512, ), eps=self.eps)
        x = (1 + gamma) * x + beta
        return x  # [1, 75, 512]


class ProsodyPredictor(nn.Module):

    def __init__(self, style_dim, d_hid, nlayers, max_dur=50):
        super().__init__()

        self.text_encoder = DurationEncoder(sty_dim=style_dim,
                                            d_model=d_hid,
                                            nlayers=nlayers)  # called outside forward
        self.lstm = nn.LSTM(d_hid + style_dim, d_hid // 2,
                            1, batch_first=True, bidirectional=True)
        self.duration_proj = LinearNorm(d_hid, max_dur)
        self.shared = nn.LSTM(d_hid + style_dim, d_hid //
                              2, 1, batch_first=True, bidirectional=True)
        self.F0 = nn.ModuleList([
            AdainResBlk1d(d_hid, d_hid, style_dim),
            AdainResBlk1d(d_hid, d_hid // 2,  style_dim, upsample=True),
            AdainResBlk1d(d_hid // 2, d_hid // 2, style_dim),
            ])
        self.N = nn.ModuleList([
            AdainResBlk1d(d_hid, d_hid, style_dim),
            AdainResBlk1d(d_hid, d_hid // 2, style_dim, upsample=True),
            AdainResBlk1d(d_hid // 2, d_hid // 2, style_dim)
            ])
        self.F0_proj = nn.Conv1d(d_hid // 2, 1, 1, 1, 0)
        self.N_proj = nn.Conv1d(d_hid // 2, 1, 1, 1, 0)

    def F0Ntrain(self, x, s):

        x, _ = self.shared(x)  # [bs, time, ch] LSTM

        x = x.transpose(1, 2)  # [bs, ch, time]

        F0 = x

        for block in self.F0:
            # print(f'LOOP {F0.shape=} {s.shape=}\n')
            # )N F0.shape=torch.Size([1, 512, 147]) s.shape=torch.Size([1, 128])
            # This is an AdainResBlk1d expects conv1d dimensions
            F0 = block(F0, s)
        F0 = self.F0_proj(F0)

        N = x

        for block in self.N:
            N = block(N, s)
        N = self.N_proj(N)

        return F0, N
    
    def forward(self, d_en=None, s=None):
        blend = self.text_encoder(d_en, s)
        x, _ = self.lstm(blend)
        dur = self.duration_proj(x)  # [bs, 150, 50]
        
        _, input_length, classifier_50 = dur.shape

        dur = dur[0, :, :]
        dur = torch.sigmoid(dur).sum(1)
        dur = dur.round().clamp(min=1).to(torch.int64)
        aln_trg = torch.zeros(1,
                              dur.sum(),
                              input_length, 
                              device=s.device)
        c_frame = 0
        for i in range(input_length):
            aln_trg[:, c_frame:c_frame + dur[i], i] = 1
            c_frame += dur[i]
        en = torch.bmm(aln_trg, blend)
        F0_pred, N_pred = self.F0Ntrain(en, s)
        return aln_trg, F0_pred, N_pred


class DurationEncoder(nn.Module):

    def __init__(self, sty_dim=128, d_model=512, nlayers=3):
        super().__init__()
        self.lstms = nn.ModuleList()
        for _ in range(nlayers):
            self.lstms.append(nn.LSTM(d_model + sty_dim,
                                      d_model // 2,
                                      num_layers=1,
                                      batch_first=True,
                                      bidirectional=True
                                      ))
            self.lstms.append(AdaLayerNorm(sty_dim, d_model))


    def forward(self, x, style):

        _, _, input_lengths = x.shape  # [bs, 512, time]

        style = _tile(style, length=x.shape[2]).transpose(1, 2)
        x = x.transpose(1, 2)

        for block in self.lstms:
            if isinstance(block, AdaLayerNorm):
                
                x = block(x, style)  # LSTM has transposed x

            else:
                x = torch.cat([x, style], axis=2)
                # LSTM

                x,_ = block(x)  # expects [bs, time, chan]  OUTPUTS [bs, time, 2*chan]  2x FROM BIDIRECTIONAL

        return torch.cat([x, style], axis=2)  # predictor.lstm()