{
  "dataset": "HateCheck (en) + Multilingual HateCheck (it, es, fr, de, pt), CC BY 4.0",
  "sample_per_language": 600,
  "seed": 42,
  "positive_class": "hateful",
  "systems": {
    "VigiMod (vigimod-1.2.1, hate)": {
      "de": {
        "n": 600,
        "precision": 0.937,
        "recall": 0.957,
        "f1": 0.947,
        "accuracy": 0.925,
        "false_positive_rate": 0.148
      },
      "en": {
        "n": 600,
        "precision": 0.952,
        "recall": 0.952,
        "f1": 0.952,
        "accuracy": 0.933,
        "false_positive_rate": 0.108
      },
      "es": {
        "n": 592,
        "precision": 0.933,
        "recall": 0.942,
        "f1": 0.938,
        "accuracy": 0.912,
        "false_positive_rate": 0.159
      },
      "fr": {
        "n": 600,
        "precision": 0.922,
        "recall": 0.9,
        "f1": 0.911,
        "accuracy": 0.877,
        "false_positive_rate": 0.178
      },
      "it": {
        "n": 600,
        "precision": 0.928,
        "recall": 0.95,
        "f1": 0.939,
        "accuracy": 0.913,
        "false_positive_rate": 0.171
      },
      "pt": {
        "n": 600,
        "precision": 0.934,
        "recall": 0.943,
        "f1": 0.938,
        "accuracy": 0.913,
        "false_positive_rate": 0.155
      },
      "all": {
        "n": 3592,
        "precision": 0.934,
        "recall": 0.941,
        "f1": 0.937,
        "accuracy": 0.912,
        "false_positive_rate": 0.153
      },
      "macro_f1": 0.938
    },
    "Detoxify multilingual (identity_attack ≥ 0.5)": {
      "de": {
        "n": 600,
        "precision": 1.0,
        "recall": 0.022,
        "f1": 0.042,
        "accuracy": 0.318,
        "false_positive_rate": 0.0
      },
      "en": {
        "n": 600,
        "precision": 0.846,
        "recall": 0.529,
        "f1": 0.651,
        "accuracy": 0.608,
        "false_positive_rate": 0.215
      },
      "es": {
        "n": 592,
        "precision": 0.836,
        "recall": 0.392,
        "f1": 0.534,
        "accuracy": 0.519,
        "false_positive_rate": 0.182
      },
      "fr": {
        "n": 600,
        "precision": 0.859,
        "recall": 0.407,
        "f1": 0.553,
        "accuracy": 0.538,
        "false_positive_rate": 0.156
      },
      "it": {
        "n": 600,
        "precision": 0.885,
        "recall": 0.406,
        "f1": 0.556,
        "accuracy": 0.548,
        "false_positive_rate": 0.122
      },
      "pt": {
        "n": 600,
        "precision": 0.864,
        "recall": 0.425,
        "f1": 0.57,
        "accuracy": 0.552,
        "false_positive_rate": 0.155
      },
      "all": {
        "n": 3592,
        "precision": 0.858,
        "recall": 0.363,
        "f1": 0.51,
        "accuracy": 0.514,
        "false_positive_rate": 0.138
      },
      "macro_f1": 0.484
    },
    "Detoxify multilingual (toxicity ≥ 0.5)": {
      "de": {
        "n": 600,
        "precision": 0.821,
        "recall": 0.055,
        "f1": 0.103,
        "accuracy": 0.333,
        "false_positive_rate": 0.027
      },
      "en": {
        "n": 600,
        "precision": 0.77,
        "recall": 0.841,
        "f1": 0.804,
        "accuracy": 0.717,
        "false_positive_rate": 0.559
      },
      "es": {
        "n": 592,
        "precision": 0.778,
        "recall": 0.808,
        "f1": 0.792,
        "accuracy": 0.703,
        "false_positive_rate": 0.545
      },
      "fr": {
        "n": 600,
        "precision": 0.79,
        "recall": 0.788,
        "f1": 0.789,
        "accuracy": 0.705,
        "false_positive_rate": 0.489
      },
      "it": {
        "n": 600,
        "precision": 0.771,
        "recall": 0.773,
        "f1": 0.772,
        "accuracy": 0.682,
        "false_positive_rate": 0.53
      },
      "pt": {
        "n": 600,
        "precision": 0.786,
        "recall": 0.79,
        "f1": 0.788,
        "accuracy": 0.703,
        "false_positive_rate": 0.497
      },
      "all": {
        "n": 3592,
        "precision": 0.779,
        "recall": 0.676,
        "f1": 0.724,
        "accuracy": 0.64,
        "false_positive_rate": 0.441
      },
      "macro_f1": 0.675
    }
  },
  "attribution": "Test cases: HateCheck (Röttger et al., 2021) and Multilingual HateCheck (Röttger et al., 2022), CC BY 4.0"
}