feature: Add dataset fetch retry and update tests with new word sets references

docs/user_guide/measurement_user_guide.rst

@@ -309,7 +309,7 @@ names with respect to pleasant and unpleasant attributes.
 
     ethnicity_query = Query(
         [word_sets["european_american_names_5"], word_sets["african_american_names_5"]],
-        [word_sets["pleasant_5"], word_sets["unpleasant_5"]],
+        [word_sets["pleasant_5"], word_sets["unpleasant_5a"]],
         ["European american names", "African american names"],
         ["Pleasant", "Unpleasant"],
     )
@@ -1265,7 +1265,7 @@ Ethnicity Bias Model Ranking
     # define the queries
     ethnicity_query_1 = Query(
         [word_sets["european_american_names_5"], word_sets["african_american_names_5"]],
-        [word_sets["pleasant_5"], word_sets["unpleasant_5"]],
+        [word_sets["pleasant_5"], word_sets["unpleasant_5a"]],
         ["European Names", "African Names"],
         ["Pleasant", "Unpleasant"],
     )

docs/user_guide/mitigation_user_guide.rst

@@ -281,7 +281,7 @@ Next, we measure the gender bias exposed by query 2 (Male Names and Female Names
 
     gender_query_2 = Query(
         [weat_wordset["male_names"], weat_wordset["female_names"]],
-        [weat_wordset["pleasant_5"], weat_wordset["unpleasant_5"]],
+        [weat_wordset["pleasant_5"], weat_wordset["unpleasant_5a"]],
         ["Male Names", "Female Names"],
         ["Pleasant", "Unpleasant"],
     )
@@ -433,7 +433,7 @@ equalized.
 
     gender_query_2 = Query(
         [weat_wordset["male_names"], weat_wordset["female_names"]],
-        [weat_wordset["pleasant_5"], weat_wordset["unpleasant_5"]],
+        [weat_wordset["pleasant_5"], weat_wordset["unpleasant_5a"]],
         ["Male Names", "Female Names"],
         ["Pleasant", "Unpleasant"],
     )

tests/debias/conftest.py

@@ -116,7 +116,7 @@ def gender_query_1(weat_wordsets: dict[str, list[str]]) -> Query:
     """
     query = Query(
         [weat_wordsets["male_names"], weat_wordsets["female_names"]],
-        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]],
+        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]],
         ["Male Names", "Female Names"],
         ["Pleasant", "Unpleasant"],
     )
@@ -202,9 +202,9 @@ def ethnicity_query_1(weat_wordsets: dict[str, list[str]]) -> Query:
             weat_wordsets["european_american_names_5"],
             weat_wordsets["african_american_names_5"],
         ],
-        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]],
+        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]],
         ["european_american_names_5", "african_american_names_5"],
-        ["pleasant_5", "unpleasant_5"],
+        ["pleasant_5", "unpleasant_5a"],
     )
     return query
 
@@ -226,8 +226,8 @@ def control_query_1(weat_wordsets: dict[str, list[str]]) -> Query:
     """
     query = Query(
         [weat_wordsets["flowers"], weat_wordsets["insects"]],
-        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]],
+        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]],
         ["flowers", "insects"],
-        ["pleasant_5", "unpleasant_5"],
+        ["pleasant_5", "unpleasant_5a"],
     )
     return query

tests/debias/test_double_hard_debias.py

@@ -170,7 +170,7 @@ def test_double_hard_debias_checks(
 #     )
 
 #     targets = weat_wordsets["male_names"] + weat_wordsets["female_names"]
-#     attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5"]
+#     attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5a"]
 #     ignore = targets + attributes
 
 #     gender_debiased_w2v = dhd.fit(

tests/debias/test_half_sibling_regression.py

@@ -108,7 +108,7 @@ def test_half_sibling_regression_ignore_param(
     )
 
     targets = weat_wordsets["male_names"] + weat_wordsets["female_names"]
-    attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5"]
+    attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5a"]
     ignore = targets + attributes
 
     gender_debiased_w2v = hsr.fit(model, definitional_words=gender_specific).transform(

tests/debias/test_hard_debias.py

@@ -148,7 +148,7 @@ def test_hard_debias_ignore_param(
     # this implies that neither of these words should be subjected to debias and
     # therefore, both queries when executed with weat should return the same score.
     targets = weat_wordsets["male_names"] + weat_wordsets["female_names"]
-    attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5"]
+    attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5a"]
     ignore = targets + attributes
 
     gender_debiased_w2v = hd.fit(

tests/debias/test_multiclass_hard_debias.py

@@ -188,7 +188,7 @@ def test_multiclass_hard_debias_ignore_param(
     # this implies that neither of these words should be subjected to debias and
     # therefore, both queries when executed with weat should return the same score.
     targets = weat_wordsets["male_names"] + weat_wordsets["female_names"]
-    attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5"]
+    attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5a"]
     ignore = targets + attributes
 
     gender_debiased_w2v = mhd.fit(

tests/metrics/conftest.py

@@ -72,7 +72,7 @@ def query_2t2a_1(weat_wordsets: dict[str, list[str]]) -> Query:
     """
     query = Query(
         [weat_wordsets["flowers"], weat_wordsets["insects"]],
-        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]],
+        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]],
         ["Flowers", "Insects"],
         ["Pleasant", "Unpleasant"],
     )
@@ -87,7 +87,7 @@ def query_3t2a_1(weat_wordsets: dict[str, list[str]]) -> Query:
             weat_wordsets["insects"],
             weat_wordsets["instruments"],
         ],
-        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]],
+        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]],
         ["Flowers", "Weapons", "Instruments"],
         ["Pleasant", "Unpleasant"],
     )
@@ -104,7 +104,7 @@ def query_4t2a_1(weat_wordsets: dict[str, list[str]]) -> Query:
             weat_wordsets["instruments"],
             weat_wordsets["weapons"],
         ],
-        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]],
+        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]],
         ["Flowers", "Insects", "Instruments", "Weapons"],
         ["Pleasant", "Unpleasant"],
     )
@@ -119,7 +119,7 @@ def query_1t4_1(weat_wordsets: dict[str, list[str]]) -> Query:
         [
             weat_wordsets["pleasant_5"],
             weat_wordsets["pleasant_9"],
-            weat_wordsets["unpleasant_5"],
+            weat_wordsets["unpleasant_5a"],
             weat_wordsets["unpleasant_9"],
         ],
         ["Flowers"],
@@ -144,7 +144,7 @@ def query_2t1a_lost_vocab_1(weat_wordsets: dict[str, list[str]]) -> Query:
 def query_2t2a_lost_vocab_1(weat_wordsets: dict[str, list[str]]) -> Query:
     query = Query(
         [["bla", "asd"], weat_wordsets["insects"]],
-        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]],
+        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]],
         ["Flowers", "Insects"],
         ["Pleasant", "Unpleasant"],
     )

tests/test_datasets.py

@@ -132,7 +132,7 @@ def test_load_weat() -> None:
         "flowers",
         "insects",
         "pleasant_5",
-        "unpleasant_5",
+        "unpleasant_5a",
         "instruments",
         "weapons",
         "european_american_names_5",

tests/test_preprocessing.py

@@ -46,7 +46,7 @@ def query_2t2a_1(weat_wordsets: dict[str, list[str]]) -> Query:
     """
     query = Query(
         [weat_wordsets["flowers"], weat_wordsets["insects"]],
-        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]],
+        [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]],
         ["Flowers", "Insects"],
         ["Pleasant", "Unpleasant"],
     )
@@ -89,7 +89,7 @@ def query_2t2a_uppercase(weat_wordsets: dict[str, list[str]]) -> Query:
         ],
         [
             [s.upper() for s in weat_wordsets["pleasant_5"]],
-            [s.upper() for s in weat_wordsets["unpleasant_5"]],
+            [s.upper() for s in weat_wordsets["unpleasant_5a"]],
         ],
         ["Flowers", "Insects"],
         ["Pleasant", "Unpleasant"],
@@ -583,7 +583,7 @@ def test_get_embeddings_from_query(
         weat_wordsets["flowers"],
         weat_wordsets["insects"],
         weat_wordsets["pleasant_5"],
-        weat_wordsets["unpleasant_5"],
+        weat_wordsets["unpleasant_5a"],
     )
 
     word_vectors = model.wv
@@ -635,7 +635,7 @@ def test_get_embeddings_from_query_oov_warns(
         weat_wordsets["flowers"],
         weat_wordsets["insects"],
         weat_wordsets["pleasant_5"],
-        weat_wordsets["unpleasant_5"],
+        weat_wordsets["unpleasant_5a"],
     )
 
     flowers_with_oov = flowers + ["aaa", "bbb"]
@@ -667,7 +667,7 @@ def test_get_embeddings_from_query_with_lower_preprocessor(
         weat_wordsets["flowers"],
         weat_wordsets["insects"],
         weat_wordsets["pleasant_5"],
-        weat_wordsets["unpleasant_5"],
+        weat_wordsets["unpleasant_5a"],
     )
 
     embeddings = get_embeddings_from_query(
@@ -706,7 +706,7 @@ def test_get_embeddings_from_query_with_two_preprocessors(
         weat_wordsets["flowers"],
         weat_wordsets["insects"],
         weat_wordsets["pleasant_5"],
-        weat_wordsets["unpleasant_5"],
+        weat_wordsets["unpleasant_5a"],
     )
     embeddings = get_embeddings_from_query(
         model, query_2t2a_uppercase, preprocessors=[{}, {"lowercase": True}]
@@ -741,7 +741,7 @@ def test_get_embeddings_from_query_lost_threshold(
         weat_wordsets["flowers"],
         weat_wordsets["insects"],
         weat_wordsets["pleasant_5"],
-        weat_wordsets["unpleasant_5"],
+        weat_wordsets["unpleasant_5a"],
     )
 
     # with lost vocabulary threshold.

tests/test_query.py

@@ -78,7 +78,7 @@ def test_create_query() -> None:
     flowers = weat_wordsets["flowers"]
     insects = weat_wordsets["insects"]
     pleasant = weat_wordsets["pleasant_5"]
-    unpleasant = weat_wordsets["unpleasant_5"]
+    unpleasant = weat_wordsets["unpleasant_5a"]
 
     # create a query using the word sets:
     query = Query(
@@ -108,7 +108,7 @@ def test_eq() -> None:
     weapons = weat["weapons"]
     pleasant_1 = weat["pleasant_5"]
     pleasant_2 = weat["pleasant_9"]
-    unpleasant_1 = weat["unpleasant_5"]
+    unpleasant_1 = weat["unpleasant_5a"]
     unpleasant_2 = weat["unpleasant_9"]
 
     query = Query([flowers, insects], [pleasant_1, unpleasant_1])
@@ -266,7 +266,7 @@ def test_generate_query_name() -> None:
 
     query = Query(
         [weat_word_set["flowers"], weat_word_set["instruments"]],
-        [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5"]],
+        [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5a"]],
         ["Flowers", "Instruments"],
         ["Pleasant", "Unpleasant"],
     )
@@ -280,7 +280,7 @@ def test_generate_query_name() -> None:
             weat_word_set["weapons"],
             weat_word_set["insects"],
         ],
-        [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5"]],
+        [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5a"]],
         ["Flowers", "Instruments", "Weapons", "Insects"],
         ["Pleasant", "Unpleasant"],
     )
@@ -297,7 +297,7 @@ def test_generate_query_name() -> None:
             weat_word_set["weapons"],
             weat_word_set["insects"],
         ],
-        [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5"]],
+        [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5a"]],
     )
 
     assert (

tests/test_utils.py

@@ -62,14 +62,14 @@ def queries_and_models():
     # Create ethnicity queries
     test_query_1 = Query(
         [word_sets["insects"], word_sets["flowers"]],
-        [word_sets["pleasant_5"], word_sets["unpleasant_5"]],
+        [word_sets["pleasant_5"], word_sets["unpleasant_5a"]],
         ["Flowers", "Insects"],
         ["Pleasant", "Unpleasant"],
     )
 
     test_query_2 = Query(
         [word_sets["weapons"], word_sets["instruments"]],
-        [word_sets["pleasant_5"], word_sets["unpleasant_5"]],
+        [word_sets["pleasant_5"], word_sets["unpleasant_5a"]],
         ["Instruments", "Weapons"],
         ["Pleasant", "Unpleasant"],
     )