Corrections to the PeerQA and DeepPlanning scores of WeMM models

scores/WeMM-Embedding-2B.json

@@ -4641,54 +4641,54 @@
                 "num_data": 2162
             },
             "PeerQA": {
-                "hit@1": 0.25735294117647056,
-                "hit@5": 0.5588235294117647,
-                "hit@10": 0.7205882352941176,
-                "ndcg_linear@1": 0.25735294117647056,
-                "ndcg_linear@5": 0.3056998358676784,
-                "ndcg_linear@10": 0.3636935775356273,
-                "ndcg_exponential@1": 0.25735294117647056,
-                "ndcg_exponential@5": 0.3056998358676784,
-                "ndcg_exponential@10": 0.3636935775356273,
-                "precision@1": 0.25735294117647056,
-                "precision@5": 0.13970588235294118,
-                "precision@10": 0.1088235294117647,
-                "recall@1": 0.1696778711484594,
-                "recall@5": 0.35372544122544125,
-                "recall@10": 0.4968470255234961,
-                "f1@1": 0.18961834733893557,
-                "f1@5": 0.18346198327815977,
-                "f1@10": 0.1664763022382061,
-                "map@1": 0.25735294117647056,
-                "map@5": 0.24915441176470582,
-                "map@10": 0.27679130680049796,
-                "mrr": 0.3924797560900438,
+                "hit@1": 0.17647058823529413,
+                "hit@5": 0.4485294117647059,
+                "hit@10": 0.5882352941176471,
+                "ndcg_linear@1": 0.17647058823529413,
+                "ndcg_linear@5": 0.23276895663820216,
+                "ndcg_linear@10": 0.27997484682365625,
+                "ndcg_exponential@1": 0.17647058823529413,
+                "ndcg_exponential@5": 0.23276895663820216,
+                "ndcg_exponential@10": 0.27997484682365625,
+                "precision@1": 0.17647058823529413,
+                "precision@5": 0.10882352941176471,
+                "precision@10": 0.08676470588235295,
+                "recall@1": 0.10693277310924369,
+                "recall@5": 0.282092172533349,
+                "recall@10": 0.3983701347671936,
+                "f1@1": 0.12221638655462184,
+                "f1@5": 0.1421629636703166,
+                "f1@10": 0.13180131027515188,
+                "map@1": 0.17647058823529413,
+                "map@5": 0.1829248366013072,
+                "map@10": 0.20505423002534348,
+                "mrr": 0.3054171260072222,
                 "num_pred": 136,
                 "num_data": 136
             },
             "DeepPlanning": {
-                "hit@1": 0.825,
-                "hit@5": 0.9833333333333333,
-                "hit@10": 0.9916666666666667,
-                "ndcg_linear@1": 0.825,
-                "ndcg_linear@5": 0.5567904065311103,
-                "ndcg_linear@10": 0.6124785543156093,
-                "ndcg_exponential@1": 0.825,
-                "ndcg_exponential@5": 0.5567904065311103,
-                "ndcg_exponential@10": 0.6124785543156093,
-                "precision@1": 0.825,
-                "precision@5": 0.4033333333333334,
-                "precision@10": 0.25666666666666665,
-                "recall@1": 0.2013888888888889,
-                "recall@5": 0.48208333333333336,
-                "recall@10": 0.6068055555555555,
-                "f1@1": 0.32075396825396824,
-                "f1@5": 0.43290404040404046,
-                "f1@10": 0.3561675824175824,
-                "map@1": 0.825,
-                "map@5": 0.41765509259259254,
-                "map@10": 0.45694422398589063,
-                "mrr": 0.8927020202020202,
+                "hit@1": 0.43333333333333335,
+                "hit@5": 0.675,
+                "hit@10": 0.7916666666666666,
+                "ndcg_linear@1": 0.43333333333333335,
+                "ndcg_linear@5": 0.2564718698410821,
+                "ndcg_linear@10": 0.2794053860266381,
+                "ndcg_exponential@1": 0.43333333333333335,
+                "ndcg_exponential@5": 0.2564718698410821,
+                "ndcg_exponential@10": 0.2794053860266381,
+                "precision@1": 0.43333333333333335,
+                "precision@5": 0.17333333333333334,
+                "precision@10": 0.11000000000000001,
+                "recall@1": 0.10722222222222223,
+                "recall@5": 0.21000000000000002,
+                "recall@10": 0.2615277777777778,
+                "f1@1": 0.17043650793650794,
+                "f1@5": 0.1872727272727273,
+                "f1@10": 0.15299068986568984,
+                "map@1": 0.43333333333333335,
+                "map@5": 0.1654884259259259,
+                "map@10": 0.17328565917107586,
+                "mrr": 0.5485158676527203,
                 "num_pred": 120,
                 "num_data": 120
             }

scores/WeMM-Embedding-4B.json

@@ -4641,54 +4641,54 @@
                 "num_data": 2162
             },
             "PeerQA": {
-                "hit@1": 0.4264705882352941,
-                "hit@5": 0.7867647058823529,
-                "hit@10": 0.8970588235294118,
-                "ndcg_linear@1": 0.4264705882352941,
-                "ndcg_linear@5": 0.49166036324392387,
-                "ndcg_linear@10": 0.5613224684920904,
-                "ndcg_exponential@1": 0.4264705882352941,
-                "ndcg_exponential@5": 0.49166036324392387,
-                "ndcg_exponential@10": 0.5613224684920904,
-                "precision@1": 0.4264705882352941,
-                "precision@5": 0.2426470588235294,
-                "precision@10": 0.175,
-                "recall@1": 0.25797982654600304,
-                "recall@5": 0.5360664458090928,
-                "recall@10": 0.7153182601712014,
-                "f1@1": 0.29488265002970887,
-                "f1@5": 0.30301042911337034,
-                "f1@10": 0.2613359237527095,
-                "map@1": 0.4264705882352941,
-                "map@5": 0.41707924836601307,
-                "map@10": 0.4563868255635587,
-                "mrr": 0.5817403688816208,
+                "hit@1": 0.3382352941176471,
+                "hit@5": 0.6911764705882353,
+                "hit@10": 0.8235294117647058,
+                "ndcg_linear@1": 0.3382352941176471,
+                "ndcg_linear@5": 0.3966662159789453,
+                "ndcg_linear@10": 0.45757569928093617,
+                "ndcg_exponential@1": 0.3382352941176471,
+                "ndcg_exponential@5": 0.3966662159789453,
+                "ndcg_exponential@10": 0.45757569928093617,
+                "precision@1": 0.3382352941176471,
+                "precision@5": 0.19999999999999998,
+                "precision@10": 0.14632352941176469,
+                "recall@1": 0.1974406108597285,
+                "recall@5": 0.4497044009911657,
+                "recall@10": 0.6056106271179801,
+                "f1@1": 0.2274806892453951,
+                "f1@5": 0.25104984884396647,
+                "f1@10": 0.21872077986161406,
+                "map@1": 0.3382352941176471,
+                "map@5": 0.3241462418300654,
+                "map@10": 0.3535050964830377,
+                "mrr": 0.49090509497394264,
                 "num_pred": 136,
                 "num_data": 136
             },
             "DeepPlanning": {
-                "hit@1": 0.7833333333333333,
-                "hit@5": 0.9916666666666667,
-                "hit@10": 0.9916666666666667,
-                "ndcg_linear@1": 0.7833333333333333,
-                "ndcg_linear@5": 0.5463660471889663,
-                "ndcg_linear@10": 0.6052911382564152,
-                "ndcg_exponential@1": 0.7833333333333333,
-                "ndcg_exponential@5": 0.5463660471889663,
-                "ndcg_exponential@10": 0.6052911382564152,
-                "precision@1": 0.7833333333333333,
-                "precision@5": 0.4016666666666667,
-                "precision@10": 0.26,
-                "recall@1": 0.19291666666666665,
-                "recall@5": 0.47875,
-                "recall@10": 0.6133333333333333,
-                "f1@1": 0.3065873015873016,
-                "f1@5": 0.4304124579124579,
-                "f1@10": 0.3606593406593406,
-                "map@1": 0.7833333333333333,
-                "map@5": 0.40427314814814813,
-                "map@10": 0.4440738536155202,
-                "mrr": 0.8680952380952381,
+                "hit@1": 0.425,
+                "hit@5": 0.6833333333333333,
+                "hit@10": 0.7833333333333333,
+                "ndcg_linear@1": 0.425,
+                "ndcg_linear@5": 0.25510983388462605,
+                "ndcg_linear@10": 0.28178380314826973,
+                "ndcg_exponential@1": 0.425,
+                "ndcg_exponential@5": 0.25510983388462605,
+                "ndcg_exponential@10": 0.28178380314826973,
+                "precision@1": 0.425,
+                "precision@5": 0.1716666666666667,
+                "precision@10": 0.11250000000000002,
+                "recall@1": 0.10555555555555557,
+                "recall@5": 0.20763888888888887,
+                "recall@10": 0.26652777777777775,
+                "f1@1": 0.16753968253968252,
+                "f1@5": 0.18516414141414145,
+                "f1@10": 0.1562171855921856,
+                "map@1": 0.425,
+                "map@5": 0.1632199074074074,
+                "map@10": 0.17248578042328042,
+                "mrr": 0.551455788715978,
                 "num_pred": 120,
                 "num_data": 120
             }

scores/WeMM-Embedding-9B.json

@@ -4641,54 +4641,54 @@
                 "num_data": 2162
             },
             "PeerQA": {
-                "hit@1": 0.35294117647058826,
-                "hit@5": 0.7279411764705882,
-                "hit@10": 0.8823529411764706,
-                "ndcg_linear@1": 0.35294117647058826,
-                "ndcg_linear@5": 0.4423072306990305,
-                "ndcg_linear@10": 0.5185990095536844,
-                "ndcg_exponential@1": 0.35294117647058826,
-                "ndcg_exponential@5": 0.4423072306990305,
-                "ndcg_exponential@10": 0.5185990095536844,
-                "precision@1": 0.35294117647058826,
-                "precision@5": 0.22058823529411764,
-                "precision@10": 0.16544117647058823,
-                "recall@1": 0.1996442895339954,
-                "recall@5": 0.5144691705353469,
-                "recall@10": 0.7027482321599968,
-                "f1@1": 0.23500233426704012,
-                "f1@5": 0.28073078881902414,
-                "f1@10": 0.25037965587602645,
-                "map@1": 0.35294117647058826,
-                "map@5": 0.3626184640522876,
-                "map@10": 0.4060355461629096,
-                "mrr": 0.5245513217228319,
+                "hit@1": 0.29411764705882354,
+                "hit@5": 0.6397058823529411,
+                "hit@10": 0.75,
+                "ndcg_linear@1": 0.29411764705882354,
+                "ndcg_linear@5": 0.35947348254787254,
+                "ndcg_linear@10": 0.4120516129270667,
+                "ndcg_exponential@1": 0.29411764705882354,
+                "ndcg_exponential@5": 0.35947348254787254,
+                "ndcg_exponential@10": 0.4120516129270667,
+                "precision@1": 0.29411764705882354,
+                "precision@5": 0.18235294117647058,
+                "precision@10": 0.1323529411764706,
+                "recall@1": 0.16018350522026992,
+                "recall@5": 0.412140739162798,
+                "recall@10": 0.5417800949050949,
+                "f1@1": 0.1908846872082166,
+                "f1@5": 0.22816840022722376,
+                "f1@10": 0.1976229966855437,
+                "map@1": 0.29411764705882354,
+                "map@5": 0.28811274509803925,
+                "map@10": 0.31813349645413724,
+                "mrr": 0.44838899055950593,
                 "num_pred": 136,
                 "num_data": 136
             },
             "DeepPlanning": {
-                "hit@1": 0.7833333333333333,
-                "hit@5": 0.975,
-                "hit@10": 0.9833333333333333,
-                "ndcg_linear@1": 0.7833333333333333,
-                "ndcg_linear@5": 0.5427052404485176,
-                "ndcg_linear@10": 0.6053325825139879,
-                "ndcg_exponential@1": 0.7833333333333333,
-                "ndcg_exponential@5": 0.5427052404485176,
-                "ndcg_exponential@10": 0.6053325825139879,
-                "precision@1": 0.7833333333333333,
-                "precision@5": 0.3983333333333334,
-                "precision@10": 0.25916666666666666,
-                "recall@1": 0.19208333333333333,
-                "recall@5": 0.47069444444444447,
-                "recall@10": 0.6141666666666666,
-                "f1@1": 0.3055952380952381,
-                "f1@5": 0.4251557239057239,
-                "f1@10": 0.3600167887667887,
-                "map@1": 0.7833333333333333,
-                "map@5": 0.4029907407407407,
-                "map@10": 0.44662384259259263,
-                "mrr": 0.8650160256410256,
+                "hit@1": 0.43333333333333335,
+                "hit@5": 0.7666666666666667,
+                "hit@10": 0.8,
+                "ndcg_linear@1": 0.43333333333333335,
+                "ndcg_linear@5": 0.2757164424964664,
+                "ndcg_linear@10": 0.29888528182940827,
+                "ndcg_exponential@1": 0.43333333333333335,
+                "ndcg_exponential@5": 0.2757164424964664,
+                "ndcg_exponential@10": 0.29888528182940827,
+                "precision@1": 0.43333333333333335,
+                "precision@5": 0.19333333333333336,
+                "precision@10": 0.12166666666666667,
+                "recall@1": 0.10180555555555555,
+                "recall@5": 0.22722222222222221,
+                "recall@10": 0.28152777777777777,
+                "f1@1": 0.1634126984126984,
+                "f1@5": 0.20586279461279464,
+                "f1@10": 0.16777014652014652,
+                "map@1": 0.43333333333333335,
+                "map@5": 0.17428703703703707,
+                "map@10": 0.18372674162257496,
+                "mrr": 0.5758292083606976,
                 "num_pred": 120,
                 "num_data": 120
             }