Hi, not sure if this question goes here or spaCy support, so here goes.
Prodigy version: 1.11.2
Training command: prodigy train ner_20210830 --ner <task_name> --base-model en_core_web_trf --label-stats
The training went great, the model has an F-score around 0.8. When trying to load it with spacy.load, I get a config error due to the following configuration block in the auto-generated config.cfg file in model-best (offending line is the one setting grad_factor):
[components.tagger.model.tok2vec]
@architectures = "spacy-transformers.Tok2VecTransformer.v1"
name = "roberta-base"
pooling = {"@layers":"reduce_mean.v1"}
grad_factor = {"@layers":"reduce_mean.v1"}
Obviously this is some kind of error as Tok2VecTransformer.v1 expects a float for grad_factor, and the model won't load. I tried changing this by hand to 1.0, but that's generating the following error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
/var/folders/30/g5wyp_x905l0_06s76c87yx40000gp/T/ipykernel_75583/445984189.py in <module>
----> 1 nlp = spacy.load('ner_20210830/model-best/')
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/spacy/__init__.py in load(name, vocab, disable, exclude, config)
49 RETURNS (Language): The loaded nlp object.
50 """
---> 51 return util.load_model(
52 name, vocab=vocab, disable=disable, exclude=exclude, config=config
53 )
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/spacy/util.py in load_model(name, vocab, disable, exclude, config)
321 return load_model_from_package(name, **kwargs)
322 if Path(name).exists(): # path to model data directory
--> 323 return load_model_from_path(Path(name), **kwargs)
324 elif hasattr(name, "exists"): # Path or Path-like to model data
325 return load_model_from_path(name, **kwargs)
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/spacy/util.py in load_model_from_path(model_path, meta, vocab, disable, exclude, config)
388 config = load_config(config_path, overrides=overrides)
389 nlp = load_model_from_config(config, vocab=vocab, disable=disable, exclude=exclude)
--> 390 return nlp.from_disk(model_path, exclude=exclude, overrides=overrides)
391
392
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/spacy/language.py in from_disk(self, path, exclude, overrides)
1965 # Convert to list here in case exclude is (default) tuple
1966 exclude = list(exclude) + ["vocab"]
-> 1967 util.from_disk(path, deserializers, exclude)
1968 self._path = path
1969 self._link_components()
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/spacy/util.py in from_disk(path, readers, exclude)
1197 # Split to support file names like meta.json
1198 if key.split(".")[0] not in exclude:
-> 1199 reader(path / key)
1200 return path
1201
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/spacy/language.py in <lambda>(p, proc)
1959 if not hasattr(proc, "from_disk"):
1960 continue
-> 1961 deserializers[name] = lambda p, proc=proc: proc.from_disk(
1962 p, exclude=["vocab"]
1963 )
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/spacy/pipeline/trainable_pipe.pyx in spacy.pipeline.trainable_pipe.TrainablePipe.from_disk()
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/spacy/util.py in from_disk(path, readers, exclude)
1197 # Split to support file names like meta.json
1198 if key.split(".")[0] not in exclude:
-> 1199 reader(path / key)
1200 return path
1201
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/spacy/pipeline/trainable_pipe.pyx in spacy.pipeline.trainable_pipe.TrainablePipe.from_disk.load_model()
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/spacy/pipeline/trainable_pipe.pyx in spacy.pipeline.trainable_pipe.TrainablePipe.from_disk.load_model()
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/thinc/model.py in from_bytes(self, bytes_data)
527 msg = srsly.msgpack_loads(bytes_data)
528 msg = convert_recursive(is_xp_array, self.ops.asarray, msg)
--> 529 return self.from_dict(msg)
530
531 def from_disk(self, path: Union[Path, str]) -> "Model":
~/dev/proj/spacy_ner/env/lib/python3.8/site-packages/thinc/model.py in from_dict(self, msg)
544 nodes = list(self.walk())
545 if len(msg["nodes"]) != len(nodes):
--> 546 raise ValueError("Cannot deserialize model: mismatched structure")
547 for i, node in enumerate(nodes):
548 info = msg["nodes"][i]
ValueError: Cannot deserialize model: mismatched structure
I'm mystified as to why Prodigy would have created an invalid config.cfg, and not sure how to fix this. I'd really like to use this model, especially as it took ~300 CPU hours to train up! Any help would be much appreciated. Thanks.
