I've made a draft that can be explored further, but it does involve custom javascript and HTML.
I'm using this Jinja2 template:
<button onclick="toggle('pos')">Positive</button>
<div id="pos" style="display: none;">
<form style="display: block;">
{%- for reason in options["pos"] -%}
<input type="checkbox" class="checkbox" id="{{reason}}" name="{{reason}}" onchange="update()" style="margin: 0.4rem;"><label for="{{reason}}">{{reason}}</label><br>
{%- endfor -%}
</form>
</div>
<button onclick="toggle('neg')">Negative</button>
<div id="neg" style="display: none;">
<form style="display: block;">
{%- for reason in options["neg"] -%}
<input type="checkbox" class="checkbox" id="{{reason}}" name="{{reason}}" onchange="update()" style="margin: 0.4rem;"><label for="{{reason}}">{{reason}}</label><br>
{%- endfor -%}
</form>
</div>
With this custom Javascript:
function toggle(id) {
var x = document.getElementById(id);
if (id == "pos"){
reset("neg")
}else{
reset("pos")
}
if (x.style.display === "none") {
x.style.display = "block";
} else {
x.style.display = "none";
}
}
function reset(id){
var x = document.getElementById(id);
x.style.display = "none"
var checkboxes = document.getElementsByClassName("checkbox");
for(let elem in checkboxes){
checkboxes[elem].checked = false;
}
}
function update(){
var checkboxes = document.getElementsByClassName("checkbox");
var results = [];
for(let elem in checkboxes){
if(checkboxes[elem].checked){
results.push(checkboxes[elem].id)
}
}
prodigy.update({
selected: results
})
}
document.addEventListener('prodigyanswer', event => {
reset("pos")
reset("neg")
})
I'm combining both in this custom recipe:
import jinja2
import prodigy
from typing import Union
from pathlib import Path
from prodigy.util import msg
from prodigy import set_hashes
from prodigy.components.loaders import JSONL
def load_template(path: Union[str, Path]) -> jinja2.Template:
if not isinstance(path, Path):
path = Path(path)
if not path.suffix == ".jinja2":
msg.fail(
"Must supply jinja2 file.",
exits=1,
)
with path.open("r", encoding="utf8") as file_:
text = file_.read()
return jinja2.Template(text, undefined=jinja2.DebugUndefined)
@prodigy.recipe(
"super-custom-ui",
dataset=("The dataset to use", "positional", None, str),
source=("The source data as a JSONL file", "positional", None, str),
)
def user_post_interest_likert(
dataset: str,
source: str,
):
stream = JSONL(source)
options = {
"pos": [
"polite",
"fast",
"useful"
],
"neg": [
"slow",
"unfriendly",
"wrong information"
]
}
template = load_template("template.jinja2")
def add_template(stream):
for ex in stream:
ex['html'] = template.render(options=options)
yield set_hashes(ex)
custom_js = Path("custom.js").read_text()
def before_db(examples):
for ex in examples:
del ex['html']
print(examples)
return examples
return {
"view_id": "blocks",
"dataset": dataset, # Name of dataset to save annotations
"stream": add_template(stream), # Incoming stream of examples
"config": {
"blocks": [
{"view_id": "text"},
{"view_id": "html"},
],
"javascript": custom_js,
},
"before_db": before_db
}
This results in an interface that looks like this:
You can confirm that the selections appear in the terminal. When you save, you'll trigger the before_db callback which prints output such like below.
[{
'text': 'hi my name is Johnny',
'_input_hash': -513872862,
'_task_hash': -311036035,
'_view_id': 'blocks',
'selected': ['slow', 'wrong information'],
'answer': 'accept',
'_timestamp': 1681207327
}]
Does this work? I agree that this requires a lot of custom work and it also doesn't allow one to use the keyboard shortcuts but what you suggest here isn't currently supported.
Idea
I wonder ... would an element like below suffice instead? One where you can just select and type reasons? You could still validate the data on the backend but you would be able to hide a lot of choices and maybe even some hierarchy in there. The main downside is that many options typically leads to loss of data quality, mainly because each annotator needs to be aware of what is possible and many options can lead to inconsistencies.
