{
  "domain": "life-health",
  "image": "python:3.12-slim",
  "data": "ProteinGym v1.3 substitution assays, with the MSAs and predicted structures distributed with it.",
  "benchmark": {
    "task": "Mutation effect prediction",
    "metric": {
      "name": "Spearman ρ",
      "better": "higher"
    },
    "cost": "GPU-hours",
    "subtasks": "assays",
    "guardrails": [
      "Zero-shot: no training on any ProteinGym assay labels",
      "A prediction for every mutant of every assay",
      "Public pretrained weights only"
    ],
    "aggregate": "mean",
    "baseline": {
      "name": "S2F",
      "score": 0.4556,
      "cost": 3.2,
      "subtasks": {
        "BLAT_ECOLX 1": 0.2536,
        "BLAT_ECOLX 2": 0.6676,
        "BLAT_ECOLX 3": 0.5981,
        "BLAT_ECOLX 4": 0.6003,
        "BLAT_ECOLX 5": 0.3448,
        "BLAT_ECOLX 6": 0.515,
        "BLAT_ECOLX 7": 0.5358,
        "GFP_AEQVI 1": 0.4521,
        "GFP_AEQVI 2": 0.51,
        "GFP_AEQVI 3": 0.5388,
        "GFP_AEQVI 4": 0.5616,
        "GFP_AEQVI 5": 0.3441,
        "GFP_AEQVI 6": 0.389,
        "GFP_AEQVI 7": 0.3237,
        "SPIKE_SARS2 1": 0.4682,
        "SPIKE_SARS2 2": 0.6984,
        "SPIKE_SARS2 3": 0.3714,
        "SPIKE_SARS2 4": 0.3773,
        "SPIKE_SARS2 5": 0.6591,
        "SPIKE_SARS2 6": 0.4961,
        "SPIKE_SARS2 7": 0.216,
        "P53_HUMAN 1": 0.2997,
        "P53_HUMAN 2": 0.274,
        "P53_HUMAN 3": 0.3266,
        "P53_HUMAN 4": 0.2323,
        "P53_HUMAN 5": 0.5694,
        "P53_HUMAN 6": 0.5513,
        "P53_HUMAN 7": 0.4948,
        "CAPSD_AAV2S 1": 0.5381,
        "CAPSD_AAV2S 2": 0.2725,
        "CAPSD_AAV2S 3": 0.6712,
        "CAPSD_AAV2S 4": 0.2145,
        "CAPSD_AAV2S 5": 0.2185,
        "CAPSD_AAV2S 6": 0.5027,
        "CAPSD_AAV2S 7": 0.7012,
        "TPOR_HUMAN 1": 0.2923,
        "TPOR_HUMAN 2": 0.4469,
        "TPOR_HUMAN 3": 0.6371,
        "TPOR_HUMAN 4": 0.2884,
        "TPOR_HUMAN 5": 0.2987,
        "TPOR_HUMAN 6": 0.4777,
        "TPOR_HUMAN 7": 0.2361,
        "RASK_HUMAN 1": 0.2234,
        "RASK_HUMAN 2": 0.3576,
        "RASK_HUMAN 3": 0.4791,
        "RASK_HUMAN 4": 0.5934,
        "RASK_HUMAN 5": 0.3265,
        "RASK_HUMAN 6": 0.3259,
        "RASK_HUMAN 7": 0.6642,
        "YAP1_HUMAN 1": 0.5889,
        "YAP1_HUMAN 2": 0.28,
        "YAP1_HUMAN 3": 0.5466,
        "YAP1_HUMAN 4": 0.3153,
        "YAP1_HUMAN 5": 0.3425,
        "YAP1_HUMAN 6": 0.4653,
        "YAP1_HUMAN 7": 0.658,
        "GRB2_HUMAN 1": 0.2463,
        "GRB2_HUMAN 2": 0.5134,
        "GRB2_HUMAN 3": 0.5774,
        "GRB2_HUMAN 4": 0.3414,
        "GRB2_HUMAN 5": 0.6884,
        "GRB2_HUMAN 6": 0.4577,
        "GRB2_HUMAN 7": 0.6698,
        "PABP_YEAST 1": 0.4866,
        "PABP_YEAST 2": 0.6115,
        "PABP_YEAST 3": 0.4282,
        "PABP_YEAST 4": 0.7186,
        "PABP_YEAST 5": 0.4353,
        "PABP_YEAST 6": 0.6049,
        "PABP_YEAST 7": 0.3523,
        "UBC9_HUMAN 1": 0.6498,
        "UBC9_HUMAN 2": 0.3735,
        "UBC9_HUMAN 3": 0.7093,
        "UBC9_HUMAN 4": 0.2997,
        "UBC9_HUMAN 5": 0.4436,
        "UBC9_HUMAN 6": 0.5201,
        "UBC9_HUMAN 7": 0.6788,
        "SUMO1_HUMAN 1": 0.5794,
        "SUMO1_HUMAN 2": 0.3899,
        "SUMO1_HUMAN 3": 0.5995,
        "SUMO1_HUMAN 4": 0.4154,
        "SUMO1_HUMAN 5": 0.6899,
        "SUMO1_HUMAN 6": 0.4665,
        "SUMO1_HUMAN 7": 0.4471,
        "TADBP_HUMAN 1": 0.573,
        "TADBP_HUMAN 2": 0.354,
        "TADBP_HUMAN 3": 0.6008,
        "TADBP_HUMAN 4": 0.4799,
        "TADBP_HUMAN 5": 0.6939,
        "TADBP_HUMAN 6": 0.4551,
        "TADBP_HUMAN 7": 0.5611,
        "MSH2_HUMAN 1": 0.3446,
        "MSH2_HUMAN 2": 0.402,
        "MSH2_HUMAN 3": 0.5129,
        "MSH2_HUMAN 4": 0.5978,
        "MSH2_HUMAN 5": 0.4271,
        "MSH2_HUMAN 6": 0.3389,
        "MSH2_HUMAN 7": 0.3764,
        "BRCA1_HUMAN 1": 0.5708,
        "BRCA1_HUMAN 2": 0.2806,
        "BRCA1_HUMAN 3": 0.5667,
        "BRCA1_HUMAN 4": 0.5127,
        "BRCA1_HUMAN 5": 0.5021,
        "BRCA1_HUMAN 6": 0.3397,
        "BRCA1_HUMAN 7": 0.4242,
        "PTEN_HUMAN 1": 0.4766,
        "PTEN_HUMAN 2": 0.5808,
        "PTEN_HUMAN 3": 0.2161,
        "PTEN_HUMAN 4": 0.4596,
        "PTEN_HUMAN 5": 0.4193,
        "PTEN_HUMAN 6": 0.399,
        "PTEN_HUMAN 7": 0.3698,
        "CBS_HUMAN 1": 0.2235,
        "CBS_HUMAN 2": 0.6526,
        "CBS_HUMAN 3": 0.3563,
        "CBS_HUMAN 4": 0.4642,
        "CBS_HUMAN 5": 0.4804,
        "CBS_HUMAN 6": 0.398,
        "CBS_HUMAN 7": 0.6466,
        "HMDH_HUMAN 1": 0.4165,
        "HMDH_HUMAN 2": 0.5689,
        "HMDH_HUMAN 3": 0.3324,
        "HMDH_HUMAN 4": 0.4373,
        "HMDH_HUMAN 5": 0.3387,
        "HMDH_HUMAN 6": 0.5571,
        "HMDH_HUMAN 7": 0.6973,
        "KCNH2_HUMAN 1": 0.633,
        "KCNH2_HUMAN 2": 0.4807,
        "KCNH2_HUMAN 3": 0.3397,
        "KCNH2_HUMAN 4": 0.6634,
        "KCNH2_HUMAN 5": 0.5985,
        "KCNH2_HUMAN 6": 0.5332,
        "KCNH2_HUMAN 7": 0.5878,
        "SCN5A_HUMAN 1": 0.6162,
        "SCN5A_HUMAN 2": 0.5021,
        "SCN5A_HUMAN 3": 0.6703,
        "SCN5A_HUMAN 4": 0.5183,
        "SCN5A_HUMAN 5": 0.6283,
        "SCN5A_HUMAN 6": 0.464,
        "SCN5A_HUMAN 7": 0.4092,
        "A4_HUMAN 1": 0.3828,
        "A4_HUMAN 2": 0.6724,
        "A4_HUMAN 3": 0.2401,
        "A4_HUMAN 4": 0.338,
        "A4_HUMAN 5": 0.2127,
        "A4_HUMAN 6": 0.2303,
        "A4_HUMAN 7": 0.6873,
        "ADRB2_HUMAN 1": 0.3377,
        "ADRB2_HUMAN 2": 0.2082,
        "ADRB2_HUMAN 3": 0.592,
        "ADRB2_HUMAN 4": 0.4474,
        "ADRB2_HUMAN 5": 0.3459,
        "ADRB2_HUMAN 6": 0.4951,
        "ADRB2_HUMAN 7": 0.3978,
        "AMIE_PSEAE 1": 0.2629,
        "AMIE_PSEAE 2": 0.6215,
        "AMIE_PSEAE 3": 0.5059,
        "AMIE_PSEAE 4": 0.4138,
        "AMIE_PSEAE 5": 0.2654,
        "AMIE_PSEAE 6": 0.6665,
        "AMIE_PSEAE 7": 0.2141,
        "CCDB_ECOLI 1": 0.3909,
        "CCDB_ECOLI 2": 0.6254,
        "CCDB_ECOLI 3": 0.3311,
        "CCDB_ECOLI 4": 0.6719,
        "CCDB_ECOLI 5": 0.5689,
        "CCDB_ECOLI 6": 0.391,
        "CCDB_ECOLI 7": 0.6226,
        "DLG4_HUMAN 1": 0.3729,
        "DLG4_HUMAN 2": 0.3492,
        "DLG4_HUMAN 3": 0.5589,
        "DLG4_HUMAN 4": 0.6763,
        "DLG4_HUMAN 5": 0.3533,
        "DLG4_HUMAN 6": 0.3216,
        "DLG4_HUMAN 7": 0.2876,
        "ENV_HV1B9 1": 0.4052,
        "ENV_HV1B9 2": 0.2294,
        "ENV_HV1B9 3": 0.3804,
        "ENV_HV1B9 4": 0.4802,
        "ENV_HV1B9 5": 0.5956,
        "ENV_HV1B9 6": 0.7089,
        "ENV_HV1B9 7": 0.3976,
        "HSP82_YEAST 1": 0.4253,
        "HSP82_YEAST 2": 0.6285,
        "HSP82_YEAST 3": 0.4013,
        "HSP82_YEAST 4": 0.522,
        "HSP82_YEAST 5": 0.3124,
        "HSP82_YEAST 6": 0.3171,
        "HSP82_YEAST 7": 0.6198,
        "KKA2_KLEPN 1": 0.535,
        "KKA2_KLEPN 2": 0.3055,
        "KKA2_KLEPN 3": 0.2772,
        "KKA2_KLEPN 4": 0.3444,
        "KKA2_KLEPN 5": 0.5257,
        "KKA2_KLEPN 6": 0.2626,
        "KKA2_KLEPN 7": 0.2394,
        "MK01_HUMAN 1": 0.2564,
        "MK01_HUMAN 2": 0.4869,
        "MK01_HUMAN 3": 0.237,
        "MK01_HUMAN 4": 0.4089,
        "MK01_HUMAN 5": 0.4368,
        "MK01_HUMAN 6": 0.3644,
        "MK01_HUMAN 7": 0.5316,
        "NUD15_HUMAN 1": 0.3651,
        "NUD15_HUMAN 2": 0.2514,
        "NUD15_HUMAN 3": 0.3595,
        "NUD15_HUMAN 4": 0.2516,
        "NUD15_HUMAN 5": 0.4508,
        "NUD15_HUMAN 6": 0.3712,
        "NUD15_HUMAN 7": 0.2375,
        "PA_I34A1 1": 0.6787,
        "PA_I34A1 2": 0.2086,
        "PA_I34A1 3": 0.5401,
        "PA_I34A1 4": 0.6675,
        "PA_I34A1 5": 0.6976,
        "PA_I34A1 6": 0.517,
        "PA_I34A1 7": 0.4976
      }
    }
  }
}