# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from .adaptive_softmax import AdaptiveSoftmax
from .gelu import gelu, gelu_accurate
from .layer_norm import LayerNorm
from .learned_positional_embedding import LearnedPositionalEmbedding
from .multihead_attention import MultiheadAttention
from .multihead_attention_super import MultiheadAttentionSuper
from .positional_embedding import PositionalEmbedding
from .sinusoidal_positional_embedding import SinusoidalPositionalEmbedding
from .transformer_sentence_encoder_layer import TransformerSentenceEncoderLayer
from .transformer_sentence_encoder import TransformerSentenceEncoder
from .unfold import unfold1d
from .transformer_layer import TransformerDecoderLayer, TransformerEncoderLayer

from .embedding_super import EmbeddingSuper
from .linear_super import LinearSuper
from .layer_norm import LayerNormSuper
from .switch_layer import SwitchFNN
from .thor_layer import ThorFNN

__all__ = [
    'AdaptiveSoftmax',
    'gelu',
    'gelu_accurate',
    'LayerNorm',
    'LearnedPositionalEmbedding',
    'MultiheadAttention',
    'MultiheadAttentionSuper',
    'PositionalEmbedding',
    'SinusoidalPositionalEmbedding',
    'TransformerSentenceEncoderLayer',
    'TransformerSentenceEncoder',
    'TransformerDecoderLayer',
    'TransformerEncoderLayer',
    'unfold1d',
    'EmbeddingSuper',
    'LinearSuper',
    'LayerNormSuper',
    'SwitchFNN',
    'ThorFNN'
]
