文本序列标注 · 让 AI 读懂每一个词

# === 1. JSONL → BIO 转换 ===
import json

def jsonl_to_bio(jsonl_path, bio_path):
    with open(jsonl_path) as fin, open(bio_path, 'w') as fout:
        for line in fin:
            item = json.loads(line)
            text = item['text']
            entities = item.get('entities', [])

            # 给每个字符预设 O
            labels = ['O'] * len(text)

            for ent in entities:
                start, end, label = ent['start'], ent['end'], ent['label']
                labels[start] = f'B-{label}'
                for i in range(start+1, end):
                    labels[i] = f'I-{label}'

            # 逐字输出
            for char, label in zip(text, labels):
                if char != '\n':
                    fout.write(f'{char}\t{label}\n')
            fout.write('\n')  # 句间空行

jsonl_to_bio('train.jsonl', 'train.bio')

# === 2. 检查 BIO 格式 ===
def check_bio_format(bio_path):
    errors = []
    prev_tag = 'O'
    for line in open(bio_path):
        line = line.strip()
        if not line:
            prev_tag = 'O'; continue
        char, tag = line.split('\t')
        if tag.startswith('I-') and prev_tag == 'O':
            errors.append(f'❌ I-{tag[2:]} 前是 O')
        prev_tag = tag
    return errors

# === 3. 训练 BERT-NER（HuggingFace） ===
from transformers import AutoTokenizer, AutoModelForTokenClassification
from transformers import Trainer, TrainingArguments

model_name = 'bert-base-chinese'
tokenizer = AutoTokenizer.from_pretrained(model_name)

# 标签映射
label_list = ['O', 'B-PER', 'I-PER',
              'B-ORG', 'I-ORG',
              'B-LOC', 'I-LOC']
id2label = {i: l for i, l in enumerate(label_list)}
label2id = {l: i for i, l in id2label.items()}

model = AutoModelForTokenClassification.from_pretrained(
    model_name, num_labels=len(label_list),
    id2label=id2label, label2id=label2id
)

# 训练...
training_args = TrainingArguments(
    output_dir='./ner_model',
    num_train_epochs=10, per_device_train_batch_size=16
)
trainer = Trainer(model=model, args=training_args, train_dataset=train_dataset)
trainer.train()