|
|
|
@ -18,10 +18,12 @@ import zipfile
|
|
|
|
|
from abc import ABCMeta, abstractmethod
|
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Iterable, List,
|
|
|
|
|
Literal, Optional, Sequence, Tuple, TypeVar, Union)
|
|
|
|
|
|
|
|
|
|
import numpy as np
|
|
|
|
|
from sentencepiece import SentencePieceProcessor # type: ignore
|
|
|
|
|
from typing import (IO, Any, Callable, Iterable, Literal, Optional, Sequence,
|
|
|
|
|
TypeVar, Union, List, Dict, Tuple, TYPE_CHECKING)
|
|
|
|
|
|
|
|
|
|
if TYPE_CHECKING:
|
|
|
|
|
from typing_extensions import TypeAlias
|
|
|
|
|
|
|
|
|
|