Hi.
I’ve read this thread and searched the forum without any luck. I want to exclude already annotated data when starting up a new annotation session, but I can’t get it to work. Here is my recipe:
This is my first ever recipe and I’m only just recently taken up programming in Python, so my guess is that I’m missing something very obvious.
When starting a new session I run prodigy emne my_database -F custom_recipe.py
@prodigy.recipe('emne',
dataset = prodigy.recipe_args['dataset'])
def recipe(dataset):
''''''
filename = 'emne.jsonl'
stream = custom_stream(filename)
return {
'on_load': custom_loader(filename),
'stream': stream,
'dataset': dataset,
'exclude': [dataset],
'view_id': 'choice'
}
def custom_stream(filename):
"Create custom stream"
options = [{'id': 1, 'text': 'Arbejdsmarked og beskæftigelse'},
{'id': 2, 'text': 'Børn, unge og familie'},
{'id': 3, 'text': 'Energi, forsyning og klima'},
{'id': 4, 'text': 'EU'},
{'id': 5, 'text': 'Finanssektoren'},
{'id': 6, 'text': 'Flygtninge og asyl'},
{'id': 7, 'text': 'Folkekirken og andre trossamfund'},
{'id': 8, 'text': 'Forskning'},
{'id': 9, 'text': 'Forsvar og militær'},
{'id': 10, 'text': 'Integration'},
{'id': 11, 'text': 'Internationalt samarbejde og handel'},
{'id': 12, 'text': 'Kultur'},
{'id': 13, 'text': 'Lokalforvaltning'},
{'id': 14, 'text': 'Miljø og fødevarer'},
{'id': 15, 'text': 'Miljø og natur'},
{'id': 16, 'text': 'Offentlig forvaltning'},
{'id': 17, 'text': 'Offentlige finanser'},
{'id': 18, 'text': 'Retspolitik og justitsvæsen'},
{'id': 19, 'text': 'Rigsfællesskabet'},
{'id': 20, 'text': 'Skatter og afgifter'},
{'id': 21, 'text': 'Sociale forhold'},
{'id': 22, 'text': 'Sundhed'},
{'id': 23, 'text': 'Teknologi og digitalisering'},
{'id': 24, 'text': 'Transport og infrstruktur'},
{'id': 25, 'text': 'Uddannelse'},
{'id': 26, 'text': 'Udviklingsbistand og nødhjælp'},
{'id': 27, 'text': 'Vækst og erhverv'}]
stream = prodigy.components.loaders.JSONL(filename)
for task in stream:
task['options'] = options
yield task
def custom_loader(filename):
"Create jsonl file with data to annotate. This is only relevant for the first session"
if os.path.exists(filename):
pass
else:
connection = MySQLdb.connect(host='host',user='user',passwd='passwd',port=3306,db='my_db')
connection.set_character_set('utf8mb4')
cursor = connection.cursor(MySQLdb.cursors.DictCursor)
query = '''SELECT id, titel, resume, lovnummer from Sag WHERE typeid = 3'''
cursor.execute(query)
records = cursor.fetchall()
with jsonlines.open(filename, mode='w') as writer:
for row in records:
writer.write({'text': ('TITLE:\n'+row['titel']+'\n\nRESUME:\n'+row['resume']), 'meta':{'id':row['id'], 'lovnummer':row['lovnummer']}})
connection.close()
I also really like your custom loader solution, and I can’t see anything that looks suspicious. So just to confirm: You ran the recipe, annotated a few examples, saved them to your dataset, restarted the server, and the stream started again at the beginning, with examples you had already annotated in the previous session?