以读取CSV数据为例,里面每行的数据格式为:label,text。下面的代码会得到一个IterableDataset类型对象,可用于后面的流程
import io from torchtext.utils import unicode_csv_reader from torchtext.data.datasets_utils import _RawTextIterableDataset def _create_data_from_csv(data_path): with io.open(data_path, encoding="utf8") as f: reader = unicode_csv_reader(f) for row in reader: yield int(row[0]), ' '.join(row[1:]) def _create_raw_text_iterable_dataset(description=None,path=None): iterator=_create_data_from_csv(path) length=sum(1 for _ in iterator) iterator=_create_data_from_csv(path) # print("lenght: ",length) return _RawTextIterableDataset(description=description,full_num_lines=length,iterator=iterator) train_iter = _create_raw_text_iterable_dataset(description="AG_NEWS",path="../dataset/AG_NEWS/train.csv")#
from torchtext.data import get_tokenizer from torchtext.vocab import build_vocab_from_iterator tokenizer = get_tokenizer(tokenizer='basic_english', language='en') #输入是字符串,输出是单词列表 tokens = tokenizer("You can now install TorchText using pip!") print(tokens) def yield_tokens(data_iter): for _, text in data_iter: yield tokenizer(text) vocab = build_vocab_from_iterator(yield_tokens(train_iter), specials=["<unk>"]) vocab.set_default_index(vocab["<unk>"])
注意,参数tokenizer可以是其他分词函数。如果是None,则直接按空格切分(无法剔除紧挨着单词的标点符号);如果是basic_english则会把标点符号单独拎出来,称为normalize
voc的常用属性/方法
# In[] #构建两个比较有用的lambda表达式 text_pipeline = lambda x: vocab(tokenizer(x)) label_pipeline = lambda x: int(x) - 1 print(text_pipeline('here is the an example')) print(label_pipeline('10')) from torch.utils.data import DataLoader device = torch.device("cuda" if torch.cuda.is_available() else "cpu") def collate_batch(batch): label_list, text_list, offsets = [], [], [0] for (_label, _text) in batch: label_list.append(label_pipeline(_label)) processed_text = torch.tensor(text_pipeline(_text), dtype=torch.int64) text_list.append(processed_text) offsets.append(processed_text.size(0)) label_list = torch.tensor(label_list, dtype=torch.int64) offsets = torch.tensor(offsets[:-1]).cumsum(dim=0) text_list = torch.cat(text_list) return label_list.to(device), text_list.to(device), offsets.to(device) dataloader = DataLoader(train_iter, batch_size=8, shuffle=False, collate_fn=collate_batch)