It is still raising the same error :
(venv) rel_component$ python3 -m spacy init fill-config configs/rel_joint.cfg configs/rel_joint.cfg
venv/lib/python3.10/site-packages/torch/cuda/__init__.py:546: UserWarning: Can't initialize NVML
warnings.warn("Can't initialize NVML")
Traceback (most recent call last):
File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "venv/lib/python3.10/site-packages/spacy/__main__.py", line 4, in <module>
setup_cli()
File "venv/lib/python3.10/site-packages/spacy/cli/_util.py", line 74, in setup_cli
command(prog_name=COMMAND)
File "venv/lib/python3.10/site-packages/click/core.py", line 1130, in __call__
return self.main(*args, **kwargs)
File "venv/lib/python3.10/site-packages/typer/core.py", line 778, in main
return _main(
File "venv/lib/python3.10/site-packages/typer/core.py", line 216, in _main
rv = self.invoke(ctx)
File "venv/lib/python3.10/site-packages/click/core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "venv/lib/python3.10/site-packages/click/core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "venv/lib/python3.10/site-packages/click/core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "venv/lib/python3.10/site-packages/click/core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "venv/lib/python3.10/site-packages/typer/main.py", line 683, in wrapper
return callback(**use_params) # type: ignore
File "venv/lib/python3.10/site-packages/spacy/cli/init_config.py", line 101, in init_fill_config_cli
fill_config(output_file, base_path, pretraining=pretraining, diff=diff)
File "venv/lib/python3.10/site-packages/spacy/cli/init_config.py", line 117, in fill_config
nlp = util.load_model_from_config(config, auto_fill=True, validate=False)
File "venv/lib/python3.10/site-packages/spacy/util.py", line 554, in load_model_from_config
nlp = lang_cls.from_config(
File "venv/lib/python3.10/site-packages/spacy/language.py", line 1803, in from_config
nlp.add_pipe(
File "venv/lib/python3.10/site-packages/spacy/language.py", line 786, in add_pipe
pipe_component = self.create_pipe(
File "venv/lib/python3.10/site-packages/spacy/language.py", line 660, in create_pipe
raise ValueError(err)
ValueError: [E002] Can't find factory for 'relation_extractor' for language English (en). This usually happens when spaCy calls `nlp.create_pipe` with a custom component name that's not registered on the current language class. If you're using a Transformer, make sure to install 'spacy-transformers'. If you're using a custom component, make sure you've added the decorator `@Language.component` (for function components) or `@Language.factory` (for class components).
Available factories: attribute_ruler, tok2vec, merge_noun_chunks, merge_entities, merge_subtokens, token_splitter, doc_cleaner, parser, beam_parser, lemmatizer, trainable_lemmatizer, entity_linker, ner, beam_ner, entity_ruler, tagger, morphologizer, senter, sentencizer, textcat, spancat, future_entity_ruler, span_ruler, textcat_multilabel, en.lemmatizer
Here is rel_joint.cfg :
[paths]
train = null
dev = null
vectors = null
init_tok2vec = null
[system]
seed = 342
gpu_allocator = null
[nlp]
lang = "en"
pipeline = ["tok2vec","ner","relation_extractor"]
disabled = []
before_creation = null
after_creation = null
after_pipeline_creation = null
tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
batch_size = 1000
[components]
[components.ner]
factory = "ner"
incorrect_spans_key = "incorrect_spans"
moves = null
scorer = {"@scorers":"spacy.ner_scorer.v1"}
update_with_oracle_cut_size = 100
[components.ner.model]
@architectures = "spacy.TransitionBasedParser.v2"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
use_upper = true
nO = null
[components.ner.model.tok2vec]
@architectures = "spacy.Tok2VecListener.v1"
width = 96
upstream = "*"
[components.tok2vec]
factory = "tok2vec"
[components.tok2vec.model]
@architectures = "spacy.HashEmbedCNN.v1"
pretrained_vectors = null
width = 96
depth = 2
embed_size = 2000
window_size = 1
maxout_pieces = 3
subword_features = true
[components.relation_extractor]
factory = "relation_extractor"
threshold = 0.5
[components.relation_extractor.model]
@architectures = "rel_model.v1"
[components.relation_extractor.model.create_instance_tensor]
@architectures = "rel_instance_tensor.v1"
[components.relation_extractor.model.create_instance_tensor.tok2vec]
@architectures = "spacy.Tok2VecListener.v1"
width = ${components.tok2vec.model.width}
[components.relation_extractor.model.create_instance_tensor.pooling]
@layers = "reduce_mean.v1"
[components.relation_extractor.model.create_instance_tensor.get_instances]
@misc = "rel_instance_generator.v1"
max_length = 100
[components.relation_extractor.model.classification_layer]
@architectures = "rel_classification_layer.v1"
nI = null
nO = null
[corpora]
[corpora.dev]
@readers = "spacy.Corpus.v1"
path = ${paths.dev}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[corpora.train]
@readers = "spacy.Corpus.v1"
path = ${paths.train}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[training]
seed = ${system.seed}
gpu_allocator = ${system.gpu_allocator}
dropout = 0.1
accumulate_gradient = 1
patience = 1600000
max_epochs = 0
max_steps = 10000
eval_frequency = 500
frozen_components = []
dev_corpus = "corpora.dev"
train_corpus = "corpora.train"
before_to_disk = null
annotating_components = ["ner"]
logger = {"@loggers":"spacy.ConsoleLogger.v1"}
[training.batcher]
@batchers = "spacy.batch_by_words.v1"
discard_oversize = false
tolerance = 0.2
get_length = null
[training.batcher.size]
@schedules = "compounding.v1"
start = 100
stop = 1000
compound = 1.001
[training.optimizer]
@optimizers = "Adam.v1"
beta1 = 0.9
beta2 = 0.999
L2_is_weight_decay = true
L2 = 0.01
grad_clip = 1.0
use_averages = false
eps = 0.00000001
learn_rate = 0.001
[training.score_weights]
rel_micro_p = 0.0
rel_micro_r = 0.0
rel_micro_f = 1.0
[initialize]
vectors = ${paths.vectors}
init_tok2vec = ${paths.init_tok2vec}
vocab_data = null
lookups = null
before_init = null
after_init = null
[initialize.components]
[initialize.tokenizer]
custom_functions.py :
from functools import partial
from pathlib import Path
from typing import Iterable, Callable
import spacy
from spacy.training import Example
from spacy.tokens import DocBin, Doc
# make the factory work
from scripts.rel_pipe import make_relation_extractor
# make the config work
from scripts.rel_model import create_relation_model, create_classification_layer, create_instances, create_tensors
@spacy.registry.readers("Gold_ents_Corpus.v1")
def create_docbin_reader(file: Path) -> Callable[["Language"], Iterable[Example]]:
return partial(read_files, file)
def read_files(file: Path, nlp: "Language") -> Iterable[Example]:
"""Custom reader that keeps the tokenization of the gold data,
and also adds the gold GGP annotations as we do not attempt to predict these."""
doc_bin = DocBin().from_disk(file)
docs = doc_bin.get_docs(nlp.vocab)
for gold in docs:
pred = Doc(
nlp.vocab,
words=[t.text for t in gold],
spaces=[t.whitespace_ for t in gold],
)
pred.ents = gold.ents
yield Example(pred, gold)
evaluate.py :
import random
import typer
from pathlib import Path
import spacy
from spacy.tokens import DocBin, Doc
from spacy.training.example import Example
# make the factory work
from rel_pipe import make_relation_extractor, score_relations
# make the config work
from rel_model import create_relation_model, create_classification_layer, create_instances, create_tensors
def main(trained_pipeline: Path, test_data: Path, print_details: bool):
nlp = spacy.load(trained_pipeline)
doc_bin = DocBin(store_user_data=True).from_disk(test_data)
docs = doc_bin.get_docs(nlp.vocab)
examples = []
for gold in docs:
pred = Doc(
nlp.vocab,
words=[t.text for t in gold],
spaces=[t.whitespace_ for t in gold],
)
pred.ents = gold.ents
for name, proc in nlp.pipeline:
pred = proc(pred)
examples.append(Example(pred, gold))
# Print the gold and prediction, if gold label is not 0
if print_details:
print()
print(f"Text: {gold.text}")
print(f"spans: {[(e.start, e.text, e.label_) for e in pred.ents]}")
for value, rel_dict in pred._.rel.items():
gold_labels = [k for (k, v) in gold._.rel[value].items() if v == 1.0]
if gold_labels:
print(
f" pair: {value} --> gold labels: {gold_labels} --> predicted values: {rel_dict}"
)
print()
random_examples = []
docs = doc_bin.get_docs(nlp.vocab)
for gold in docs:
pred = Doc(
nlp.vocab,
words=[t.text for t in gold],
spaces=[t.whitespace_ for t in gold],
)
pred.ents = gold.ents
relation_extractor = nlp.get_pipe("relation_extractor")
get_instances = relation_extractor.model.attrs["get_instances"]
for (e1, e2) in get_instances(pred):
offset = (e1.start, e2.start)
if offset not in pred._.rel:
pred._.rel[offset] = {}
for label in relation_extractor.labels:
pred._.rel[offset][label] = random.uniform(0, 1)
random_examples.append(Example(pred, gold))
thresholds = [0.000, 0.050, 0.100, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.99, 0.999]
print()
print("Random baseline:")
_score_and_format(random_examples, thresholds)
print()
print("Results of the trained model:")
_score_and_format(examples, thresholds)
def _score_and_format(examples, thresholds):
for threshold in thresholds:
r = score_relations(examples, threshold)
results = {k: "{:.2f}".format(v * 100) for k, v in r.items()}
print(f"threshold {'{:.2f}'.format(threshold)} \t {results}")
if __name__ == "__main__":
typer.run(main)
parse_data_generic.py (with anonymized labels) :
# This script was derived from parse_data.py but made more generic as a template for various REL parsing needs
import json
import random
import typer
from pathlib import Path
from spacy.tokens import DocBin, Doc
from spacy.vocab import Vocab
from wasabi import Printer
msg = Printer()
# TODO: define your labels used for annotation either as "symmetrical" or "directed"
SYMM_LABELS = ["D", "E"]
DIRECTED_LABELS = ["A", "B", "C"]
# TODO: define splits for train/dev/test. What is not in test or dev, will be used as train.
test_portion = 0.1#0.2
dev_portion = 0.1#0.3
# TODO: set this bool to False if you didn't annotate all relations in all sentences.
# If it's true, entities that were not annotated as related will be used as negative examples.
is_complete = False
def main(json_loc: Path, train_file: Path, dev_file: Path, test_file: Path):
"""Creating the corpus from the Prodigy annotations."""
Doc.set_extension("rel", default={})
vocab = Vocab()
docs = {"train": [], "dev": [], "test": []}
count_all = {"train": 0, "dev": 0, "test": 0}
count_pos = {"train": 0, "dev": 0, "test": 0}
with json_loc.open("r", encoding="utf8") as jsonfile:
for line in jsonfile:
example = json.loads(line)
span_starts = set()
if example["answer"] == "accept":
neg = 0
pos = 0
# Parse the tokens
words = [t["text"] for t in example["tokens"]]
spaces = [t["ws"] for t in example["tokens"]]
doc = Doc(vocab, words=words, spaces=spaces)
# Parse the entities
spans = example["spans"]
entities = []
span_end_to_start = {}
for span in spans:
entity = doc.char_span(
span["start"], span["end"], label=span["label"]
)
span_end_to_start[span["token_end"]] = span["token_start"]
entities.append(entity)
span_starts.add(span["token_start"])
if not entities:
msg.warn("Could not parse any entities from the JSON file.")
doc.ents = entities
# Parse the relations
rels = {}
for x1 in span_starts:
for x2 in span_starts:
rels[(x1, x2)] = {}
relations = example["relations"]
for relation in relations:
# Ignoring relations that are not between spans (they are annotated on the token level
if not relation["head"] in span_end_to_start or not relation["child"] in span_end_to_start:
msg.warn(f"This script only supports relationships between annotated entities.")
break
# the 'head' and 'child' annotations refer to the end token in the span
# but we want the first token
start = span_end_to_start[relation["head"]]
end = span_end_to_start[relation["child"]]
label = relation["label"]
if label not in SYMM_LABELS + DIRECTED_LABELS:
msg.warn(f"Found label '{label}' not defined in SYMM_LABELS or DIRECTED_LABELS - skipping")
break
if label not in rels[(start, end)]:
rels[(start, end)][label] = 1.0
pos += 1
if label in SYMM_LABELS:
if label not in rels[(end, start)]:
rels[(end, start)][label] = 1.0
pos += 1
# If the annotation is complete, fill in zero's where the data is missing
if is_complete:
for x1 in span_starts:
for x2 in span_starts:
for label in SYMM_LABELS + DIRECTED_LABELS:
if label not in rels[(x1, x2)]:
neg += 1
rels[(x1, x2)][label] = 0.0
doc._.rel = rels
# only keeping documents with at least 1 positive case
if pos > 0:
# create the train/dev/test split randomly
# Note that this is not good practice as instances from the same article
# may end up in different splits. Ideally, change this method to keep
# documents together in one split (as in the original parse_data.py)
if random.random() < test_portion:
docs["test"].append(doc)
count_pos["test"] += pos
count_all["test"] += pos + neg
elif random.random() < (test_portion + dev_portion):
docs["dev"].append(doc)
count_pos["dev"] += pos
count_all["dev"] += pos + neg
else:
docs["train"].append(doc)
count_pos["train"] += pos
count_all["train"] += pos + neg
docbin = DocBin(docs=docs["train"], store_user_data=True)
docbin.to_disk(train_file)
msg.info(
f"{len(docs['train'])} training sentences, "
f"{count_pos['train']}/{count_all['train']} pos instances."
)
docbin = DocBin(docs=docs["dev"], store_user_data=True)
docbin.to_disk(dev_file)
msg.info(
f"{len(docs['dev'])} dev sentences, "
f"{count_pos['dev']}/{count_all['dev']} pos instances."
)
docbin = DocBin(docs=docs["test"], store_user_data=True)
docbin.to_disk(test_file)
msg.info(
f"{len(docs['test'])} test sentences, "
f"{count_pos['test']}/{count_all['test']} pos instances."
)
if __name__ == "__main__":
typer.run(main)
rel_model.py :
from typing import List, Tuple, Callable
import spacy
from spacy.tokens import Doc, Span
from thinc.types import Floats2d, Ints1d, Ragged, cast
from thinc.api import Model, Linear, chain, Logistic
@spacy.registry.architectures("rel_model.v1")
def create_relation_model(
create_instance_tensor: Model[List[Doc], Floats2d],
classification_layer: Model[Floats2d, Floats2d],
) -> Model[List[Doc], Floats2d]:
with Model.define_operators({">>": chain}):
model = create_instance_tensor >> classification_layer
model.attrs["get_instances"] = create_instance_tensor.attrs["get_instances"]
return model
@spacy.registry.architectures("rel_classification_layer.v1")
def create_classification_layer(
nO: int = None, nI: int = None
) -> Model[Floats2d, Floats2d]:
with Model.define_operators({">>": chain}):
return Linear(nO=nO, nI=nI) >> Logistic()
@spacy.registry.misc("rel_instance_generator.v1")
def create_instances(max_length: int) -> Callable[[Doc], List[Tuple[Span, Span]]]:
def get_instances(doc: Doc) -> List[Tuple[Span, Span]]:
instances = []
for ent1 in doc.ents:
for ent2 in doc.ents:
if ent1 != ent2:
if max_length and abs(ent2.start - ent1.start) <= max_length:
instances.append((ent1, ent2))
return instances
return get_instances
@spacy.registry.architectures("rel_instance_tensor.v1")
def create_tensors(
tok2vec: Model[List[Doc], List[Floats2d]],
pooling: Model[Ragged, Floats2d],
get_instances: Callable[[Doc], List[Tuple[Span, Span]]],
) -> Model[List[Doc], Floats2d]:
return Model(
"instance_tensors",
instance_forward,
layers=[tok2vec, pooling],
refs={"tok2vec": tok2vec, "pooling": pooling},
attrs={"get_instances": get_instances},
init=instance_init,
)
def instance_forward(model: Model[List[Doc], Floats2d], docs: List[Doc], is_train: bool) -> Tuple[Floats2d, Callable]:
pooling = model.get_ref("pooling")
tok2vec = model.get_ref("tok2vec")
get_instances = model.attrs["get_instances"]
all_instances = [get_instances(doc) for doc in docs]
tokvecs, bp_tokvecs = tok2vec(docs, is_train)
ents = []
lengths = []
for doc_nr, (instances, tokvec) in enumerate(zip(all_instances, tokvecs)):
token_indices = []
for instance in instances:
for ent in instance:
token_indices.extend([i for i in range(ent.start, ent.end)])
lengths.append(ent.end - ent.start)
ents.append(tokvec[token_indices])
lengths = cast(Ints1d, model.ops.asarray(lengths, dtype="int32"))
entities = Ragged(model.ops.flatten(ents), lengths)
pooled, bp_pooled = pooling(entities, is_train)
# Reshape so that pairs of rows are concatenated
relations = model.ops.reshape2f(pooled, -1, pooled.shape[1] * 2)
def backprop(d_relations: Floats2d) -> List[Doc]:
d_pooled = model.ops.reshape2f(d_relations, d_relations.shape[0] * 2, -1)
d_ents = bp_pooled(d_pooled).data
d_tokvecs = []
ent_index = 0
for doc_nr, instances in enumerate(all_instances):
shape = tokvecs[doc_nr].shape
d_tokvec = model.ops.alloc2f(*shape)
count_occ = model.ops.alloc2f(*shape)
for instance in instances:
for ent in instance:
d_tokvec[ent.start : ent.end] += d_ents[ent_index]
count_occ[ent.start : ent.end] += 1
ent_index += ent.end - ent.start
d_tokvec /= count_occ + 0.00000000001
d_tokvecs.append(d_tokvec)
d_docs = bp_tokvecs(d_tokvecs)
return d_docs
return relations, backprop
def instance_init(model: Model, X: List[Doc] = None, Y: Floats2d = None) -> Model:
tok2vec = model.get_ref("tok2vec")
if X is not None:
tok2vec.initialize(X)
return model
rel_pipe.py :
from itertools import islice
from typing import Tuple, List, Iterable, Optional, Dict, Callable, Any
from spacy.scorer import PRFScore
from thinc.types import Floats2d
import numpy
from spacy.training.example import Example
from thinc.api import Model, Optimizer
from spacy.tokens.doc import Doc
from spacy.pipeline.trainable_pipe import TrainablePipe
from spacy.vocab import Vocab
from spacy import Language
from thinc.model import set_dropout_rate
from wasabi import Printer
Doc.set_extension("rel", default={}, force=True)
msg = Printer()
@Language.factory(
"relation_extractor",
requires=["doc.ents", "token.ent_iob", "token.ent_type"],
assigns=["doc._.rel"],
default_score_weights={
"rel_micro_p": None,
"rel_micro_r": None,
"rel_micro_f": None,
},
)
def make_relation_extractor(
nlp: Language, name: str, model: Model, *, threshold: float
):
"""Construct a RelationExtractor component."""
return RelationExtractor(nlp.vocab, model, name, threshold=threshold)
class RelationExtractor(TrainablePipe):
def __init__(
self,
vocab: Vocab,
model: Model,
name: str = "rel",
*,
threshold: float,
) -> None:
"""Initialize a relation extractor."""
self.vocab = vocab
self.model = model
self.name = name
self.cfg = {"labels": [], "threshold": threshold}
@property
def labels(self) -> Tuple[str]:
"""Returns the labels currently added to the component."""
return tuple(self.cfg["labels"])
@property
def threshold(self) -> float:
"""Returns the threshold above which a prediction is seen as 'True'."""
return self.cfg["threshold"]
def add_label(self, label: str) -> int:
"""Add a new label to the pipe."""
if not isinstance(label, str):
raise ValueError("Only strings can be added as labels to the RelationExtractor")
if label in self.labels:
return 0
self.cfg["labels"] = list(self.labels) + [label]
return 1
def __call__(self, doc: Doc) -> Doc:
"""Apply the pipe to a Doc."""
# check that there are actually any candidate instances in this batch of examples
total_instances = len(self.model.attrs["get_instances"](doc))
if total_instances == 0:
msg.info("Could not determine any instances in doc - returning doc as is.")
return doc
predictions = self.predict([doc])
self.set_annotations([doc], predictions)
return doc
def predict(self, docs: Iterable[Doc]) -> Floats2d:
"""Apply the pipeline's model to a batch of docs, without modifying them."""
get_instances = self.model.attrs["get_instances"]
total_instances = sum([len(get_instances(doc)) for doc in docs])
if total_instances == 0:
msg.info("Could not determine any instances in any docs - can not make any predictions.")
scores = self.model.predict(docs)
return self.model.ops.asarray(scores)
def set_annotations(self, docs: Iterable[Doc], scores: Floats2d) -> None:
"""Modify a batch of `Doc` objects, using pre-computed scores."""
c = 0
get_instances = self.model.attrs["get_instances"]
for doc in docs:
for (e1, e2) in get_instances(doc):
offset = (e1.start, e2.start)
if offset not in doc._.rel:
doc._.rel[offset] = {}
for j, label in enumerate(self.labels):
doc._.rel[offset][label] = scores[c, j]
c += 1
def update(
self,
examples: Iterable[Example],
*,
drop: float = 0.0,
set_annotations: bool = False,
sgd: Optional[Optimizer] = None,
losses: Optional[Dict[str, float]] = None,
) -> Dict[str, float]:
"""Learn from a batch of documents and gold-standard information,
updating the pipe's model. Delegates to predict and get_loss."""
if losses is None:
losses = {}
losses.setdefault(self.name, 0.0)
set_dropout_rate(self.model, drop)
# check that there are actually any candidate instances in this batch of examples
total_instances = 0
for eg in examples:
total_instances += len(self.model.attrs["get_instances"](eg.predicted))
if total_instances == 0:
msg.info("Could not determine any instances in doc.")
return losses
# run the model
docs = [eg.predicted for eg in examples]
predictions, backprop = self.model.begin_update(docs)
loss, gradient = self.get_loss(examples, predictions)
backprop(gradient)
if sgd is not None:
self.model.finish_update(sgd)
losses[self.name] += loss
if set_annotations:
self.set_annotations(docs, predictions)
return losses
def get_loss(self, examples: Iterable[Example], scores) -> Tuple[float, float]:
"""Find the loss and gradient of loss for the batch of documents and
their predicted scores."""
truths = self._examples_to_truth(examples)
gradient = scores - truths
mean_square_error = (gradient ** 2).sum(axis=1).mean()
return float(mean_square_error), gradient
def initialize(
self,
get_examples: Callable[[], Iterable[Example]],
*,
nlp: Language = None,
labels: Optional[List[str]] = None,
):
"""Initialize the pipe for training, using a representative set
of data examples.
"""
if labels is not None:
for label in labels:
self.add_label(label)
else:
for example in get_examples():
relations = example.reference._.rel
for indices, label_dict in relations.items():
for label in label_dict.keys():
self.add_label(label)
self._require_labels()
subbatch = list(islice(get_examples(), 10))
doc_sample = [eg.reference for eg in subbatch]
label_sample = self._examples_to_truth(subbatch)
if label_sample is None:
raise ValueError("Call begin_training with relevant entities and relations annotated in "
"at least a few reference examples!")
self.model.initialize(X=doc_sample, Y=label_sample)
def _examples_to_truth(self, examples: List[Example]) -> Optional[numpy.ndarray]:
# check that there are actually any candidate instances in this batch of examples
nr_instances = 0
for eg in examples:
nr_instances += len(self.model.attrs["get_instances"](eg.reference))
if nr_instances == 0:
return None
truths = numpy.zeros((nr_instances, len(self.labels)), dtype="f")
c = 0
for i, eg in enumerate(examples):
for (e1, e2) in self.model.attrs["get_instances"](eg.reference):
gold_label_dict = eg.reference._.rel.get((e1.start, e2.start), {})
for j, label in enumerate(self.labels):
truths[c, j] = gold_label_dict.get(label, 0)
c += 1
truths = self.model.ops.asarray(truths)
return truths
def score(self, examples: Iterable[Example], **kwargs) -> Dict[str, Any]:
"""Score a batch of examples."""
return score_relations(examples, self.threshold)
def score_relations(examples: Iterable[Example], threshold: float) -> Dict[str, Any]:
"""Score a batch of examples."""
micro_prf = PRFScore()
for example in examples:
gold = example.reference._.rel
pred = example.predicted._.rel
for key, pred_dict in pred.items():
gold_labels = [k for (k, v) in gold.get(key, {}).items() if v == 1.0]
for k, v in pred_dict.items():
if v >= threshold:
if k in gold_labels:
micro_prf.tp += 1
else:
micro_prf.fp += 1
else:
if k in gold_labels:
micro_prf.fn += 1
return {
"rel_micro_p": micro_prf.precision,
"rel_micro_r": micro_prf.recall,
"rel_micro_f": micro_prf.fscore,
}
