【ML】数据增强

1
2
pip install nlpaug==1.1.0 transformers==3.0.2 -q
pip install snorkel==0.9.8 -q

1
import nlpaug.augmenter.word as naw

1
2
3
4
# Load tokenizers and transformers
substitution = naw.ContextualWordEmbsAug(model_path="distilbert-base-uncased", action="substitute")
insertion = naw.ContextualWordEmbsAug(model_path="distilbert-base-uncased", action="insert")
text = "Conditional image generation using Variational Autoencoders and GANs."

1
hierarchical risk mapping using variational signals and gans.

1
2
# 插入
insertion.augment(text)

1
automated conditional inverse image generation algorithms using multiple variational autoencoders and gans.

1
2
3
# 替换标签和别名中的短划线
def replace_dash(x):
    return x.replace("-", " ")

1
2
3
4
5
6
# 别名
aliases_by_tag = {
    "computer-vision": ["cv", "vision"],
    "mlops": ["production"],
    "natural-language-processing": ["nlp", "nlproc"]
}

1
2
3
4
5
6
7
8
9
10
# 扁平化字典
flattened_aliases = {}
for tag, aliases in aliases_by_tag.items():
    tag = replace_dash(x=tag)
    if len(aliases):
        flattened_aliases[tag] = aliases
    for alias in aliases:
        _aliases = aliases + [tag]
        _aliases.remove(alias)
        flattened_aliases[alias] = _aliases

1
2
print (flattened_aliases["natural language processing"])
print (flattened_aliases["nlp"])

1
2
['nlp', 'nlproc']
['nlproc', 'natural language processing']

1
2
3
# 我们只想与整个单词匹配
print ("gan" in "This is a gan.")
print ("gan" in "This is gandalf.")

1
2
3
4
5
# b matches spaces
def find_word(word, text):
    word = word.replace("+", "+")
    pattern = re.compile(fr"b({word})b", flags=re.IGNORECASE)
    return pattern.search(text)

1
2
3
# Correct behavior (single instance)
print (find_word("gan", "This is a gan."))
print (find_word("gan", "This is gandalf."))

1
2
<re.Match object; span=(10, 13), match='gan'>
None

1
from snorkel.augmentation import transformation_function

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
@transformation_function()
def swap_aliases(x):
    """Swap ML keywords with their aliases."""
    # Find all matches
    matches = []
    for i, tag in enumerate(flattened_aliases):
        match = find_word(tag, x.text)
        if match:
            matches.append(match)
    # Swap a random match with a random alias
    if len(matches):
        match = random.choice(matches)
        tag = x.text[match.start():match.end()]
        x.text = f"{x.text[:match.start()]}{random.choice(flattened_aliases[tag])}{x.text[match.end():]}"
    return x

1
2
3
4
5
# Swap
for i in range(3):
    sample_df = pd.DataFrame([{"text": "a survey of reinforcement learning for nlp tasks."}])
    sample_df.text = sample_df.text.apply(preprocess, lower=True, stem=False)
    print (swap_aliases(sample_df.iloc[0]).text)

1
2
3
4
5
# Undesired behavior (needs contextual insight)
for i in range(3):
    sample_df = pd.DataFrame([{"text": "Autogenerate your CV to apply for jobs using NLP."}])
    sample_df.text = sample_df.text.apply(preprocess, lower=True, stem=False)
    print (swap_aliases(sample_df.iloc[0]).text)

1
2
3
autogenerate vision apply jobs using nlp
autogenerate cv apply jobs using natural language processing
autogenerate cv apply jobs using nlproc

1
from snorkel.augmentation import ApplyOnePolicy, PandasTFApplier

1
2
3
4
5
6
# 转换函数 （TF） 策略
policy = ApplyOnePolicy(n_per_original=5, keep_original=True)
tf_applier = PandasTFApplier([swap_aliases], policy)
train_df_augmented = tf_applier.apply(train_df)
train_df_augmented.drop_duplicates(subset=["text"], inplace=True)
train_df_augmented.head()

1
len(train_df), len(train_df_augmented)

1
(668, 913)