@fhamborg I've come up with a solution to enforce your constraints. I tried to make it work with the current prodigy version but it required client modifications, so it should be available with the next release.
The way it works is that you specify a custom repeating feed class that causes the stream to start over at the last question that was answered each time you ask for questions. Then you set the client option to tell the server about any answered questions the client has that are undelivered. The effect is that every time you get the same questions in the same order until you answer them, even if you close the tab or hit refresh: http://www.giphy.com/gifs/iGeg41sdvj21e1a4f1
The updates to your recipe would then look something like this:
import prodigy
from prodigy.components.db import connect
from prodigy.components.feeds import RepeatingFeed
from prodigy.components.loaders import JSONL
@prodigy.recipe(
"sentiment_choices",
dataset=prodigy.recipe_args["dataset"],
file_path=("Path to texts", "positional", None, str),
)
def sentiment(dataset, file_path):
"""Annotate the sentiment of texts using different mood options."""
stream = JSONL(file_path) # load in the JSONL file
stream = add_options(stream) # add options to each task
DB = prodigy.components.db.connect()
return {
"dataset": dataset, # save annotations in this dataset
"view_id": "choice", # use the choice interface
"config": {
"choice_auto_accept": True, # auto-accept example, once the users selects an option,
"hint_pending_answers": True, # don't send questions we've already answered but haven't returned
"feed_class": RepeatingFeed, # repeat previously sent questions
"feed_filters": [],
},
"on_exit": on_exit,
"stream": stream,
}
def add_options(stream):
"""Helper function to add options to every task in a stream."""
options = [
{"id": "positive", "text": "😊 positive"},
{"id": "neutral", "text": "😶 neutral"},
{"id": "negative", "text": "🙁 negative"},
{"id": "posneg", "text": "😊+🙁 pos. and neg."},
]
for task in stream:
task["options"] = options
yield task
def on_exit(controller):
"""Get all annotations in the dataset, filter out the accepted tasks,
count them by the selected options and print the counts."""
# taken from https://prodi.gy/docs/workflow-custom-recipes#example-choice
examples = controller.db.get_dataset(controller.dataset)
examples = [eg for eg in examples if eg["answer"] == "accept"]
for option in ("positive", "neutral", "negative", "posneg"):
count = get_count_by_option(examples, option)
print("Annotated {} {} examples".format(count, option))
def get_count_by_option(examples, option):
filtered = [eg for eg in examples if option in eg["answer"]]
return len(filtered)