Changes

              
    

          
          
        
        

            
f
1
{
f
1
{
            
2
  "access_rights": "",
2
  "access_rights": "",
            
3
  "author": "Weixiang Yan",
3
  "author": "Weixiang Yan",
            
4
  "author_email": "",
4
  "author_email": "",
            
5
  "citation": [],
5
  "citation": [],
            
6
  "creator_user_id": "17755db4-395a-4b3b-ac09-e8e3484ca700",
6
  "creator_user_id": "17755db4-395a-4b3b-ac09-e8e3484ca700",
            
7
  "defined_in": "https://doi.org/10.48550/arXiv.2406.13890",
7
  "defined_in": "https://doi.org/10.48550/arXiv.2406.13890",
            
8
  "doi": "10.57702/bo4xy1bt",
8
  "doi": "10.57702/bo4xy1bt",
            
n
9
  "doi_date_published": null,
n
9
  "doi_date_published": "2024-12-03",
            
10
  "doi_publisher": "TIB",
10
  "doi_publisher": "TIB",
            
n
11
  "doi_status": false,
n
11
  "doi_status": true,
            
12
  "domain": "https://service.tib.eu/ldmservice",
12
  "domain": "https://service.tib.eu/ldmservice",
            
13
  "extra_authors": [
13
  "extra_authors": [
            
14
    {
14
    {
            
15
      "extra_author": "Haitian Liu",
15
      "extra_author": "Haitian Liu",
            
16
      "orcid": ""
16
      "orcid": ""
            
17
    },
17
    },
            
18
    {
18
    {
            
19
      "extra_author": "Tengxiao Wu",
19
      "extra_author": "Tengxiao Wu",
            
20
      "orcid": ""
20
      "orcid": ""
            
21
    },
21
    },
            
22
    {
22
    {
            
23
      "extra_author": "Qian Chen",
23
      "extra_author": "Qian Chen",
            
24
      "orcid": ""
24
      "orcid": ""
            
25
    },
25
    },
            
26
    {
26
    {
            
27
      "extra_author": "Wen Wang",
27
      "extra_author": "Wen Wang",
            
28
      "orcid": ""
28
      "orcid": ""
            
29
    },
29
    },
            
30
    {
30
    {
            
31
      "extra_author": "Haoyuan Chai",
31
      "extra_author": "Haoyuan Chai",
            
32
      "orcid": ""
32
      "orcid": ""
            
33
    },
33
    },
            
34
    {
34
    {
            
35
      "extra_author": "Jiayi Wang",
35
      "extra_author": "Jiayi Wang",
            
36
      "orcid": ""
36
      "orcid": ""
            
37
    },
37
    },
            
38
    {
38
    {
            
39
      "extra_author": "Weishan Zhao",
39
      "extra_author": "Weishan Zhao",
            
40
      "orcid": ""
40
      "orcid": ""
            
41
    },
41
    },
            
42
    {
42
    {
            
43
      "extra_author": "Yixin Zhang",
43
      "extra_author": "Yixin Zhang",
            
44
      "orcid": ""
44
      "orcid": ""
            
45
    },
45
    },
            
46
    {
46
    {
            
47
      "extra_author": "Renjun Zhang",
47
      "extra_author": "Renjun Zhang",
            
48
      "orcid": ""
48
      "orcid": ""
            
49
    },
49
    },
            
50
    {
50
    {
            
51
      "extra_author": "Li Zhu",
51
      "extra_author": "Li Zhu",
            
52
      "orcid": ""
52
      "orcid": ""
            
53
    }
53
    }
            
54
  ],
54
  ],
            
55
  "groups": [
55
  "groups": [
            
56
    {
56
    {
            
57
      "description": "",
57
      "description": "",
            
58
      "display_name": "Clinical Diagnosis",
58
      "display_name": "Clinical Diagnosis",
            
59
      "id": "a03fe69a-6a06-440b-8aee-80cbdeeab5b3",
59
      "id": "a03fe69a-6a06-440b-8aee-80cbdeeab5b3",
            
60
      "image_display_url": "",
60
      "image_display_url": "",
            
61
      "name": "clinical-diagnosis",
61
      "name": "clinical-diagnosis",
            
62
      "title": "Clinical Diagnosis"
62
      "title": "Clinical Diagnosis"
            
63
    },
63
    },
            
64
    {
64
    {
            
65
      "description": "",
65
      "description": "",
            
66
      "display_name": "Medical Imaging",
66
      "display_name": "Medical Imaging",
            
67
      "id": "b86e8f52-a230-44ce-b290-7823c9f6a877",
67
      "id": "b86e8f52-a230-44ce-b290-7823c9f6a877",
            
68
      "image_display_url": "",
68
      "image_display_url": "",
            
69
      "name": "medical-imaging",
69
      "name": "medical-imaging",
            
70
      "title": "Medical Imaging"
70
      "title": "Medical Imaging"
            
71
    },
71
    },
            
72
    {
72
    {
            
73
      "description": "",
73
      "description": "",
            
74
      "display_name": "Natural Language Processing",
74
      "display_name": "Natural Language Processing",
            
75
      "id": "5b974bcc-8f79-40fc-a05d-75b861d55ed0",
75
      "id": "5b974bcc-8f79-40fc-a05d-75b861d55ed0",
            
76
      "image_display_url": "",
76
      "image_display_url": "",
            
77
      "name": "natural-language-processing",
77
      "name": "natural-language-processing",
            
78
      "title": "Natural Language Processing"
78
      "title": "Natural Language Processing"
            
79
    }
79
    }
            
80
  ],
80
  ],
            
81
  "id": "8a350f8a-13df-4d14-9124-09ebe1867998",
81
  "id": "8a350f8a-13df-4d14-9124-09ebe1867998",
            
82
  "isopen": false,
82
  "isopen": false,
            
83
  "landing_page": "",
83
  "landing_page": "",
            
84
  "license_title": null,
84
  "license_title": null,
            
85
  "link_orkg": "",
85
  "link_orkg": "",
            
86
  "metadata_created": "2024-12-03T10:02:04.829976",
86
  "metadata_created": "2024-12-03T10:02:04.829976",
            
n
87
  "metadata_modified": "2024-12-03T10:02:04.829981",
n
87
  "metadata_modified": "2024-12-03T10:02:05.180936",
            
88
  "name": 
88
  "name": 
            
89
linicallab--a-comprehensive-clinical-diagnosis-agent-alignment-suite",
89
linicallab--a-comprehensive-clinical-diagnosis-agent-alignment-suite",
            
90
  "notes": "Large language models (LLMs) have achieved significant 
90
  "notes": "Large language models (LLMs) have achieved significant 
            
91
performance progress in various natural language processing 
91
performance progress in various natural language processing 
            
92
applications. However, LLMs still struggle to meet the strict 
92
applications. However, LLMs still struggle to meet the strict 
            
93
requirements for accuracy and reliability in the medical field and 
93
requirements for accuracy and reliability in the medical field and 
            
94
face many challenges in clinical applications. Existing clinical 
94
face many challenges in clinical applications. Existing clinical 
            
95
diagnostic evaluation benchmarks for evaluating medical agents powered 
95
diagnostic evaluation benchmarks for evaluating medical agents powered 
            
96
by LLMs have severe limitations. Firstly, most existing medical 
96
by LLMs have severe limitations. Firstly, most existing medical 
            
97
evaluation benchmarks face the risk of data leakage or contamination. 
97
evaluation benchmarks face the risk of data leakage or contamination. 
            
98
Secondly, existing benchmarks often neglect the characteristics of 
98
Secondly, existing benchmarks often neglect the characteristics of 
            
99
multiple departments and specializations in modern medical practice. 
99
multiple departments and specializations in modern medical practice. 
            
100
Thirdly, existing evaluation methods are limited to multiple-choice 
100
Thirdly, existing evaluation methods are limited to multiple-choice 
            
101
questions, which do not align with the real-world diagnostic 
101
questions, which do not align with the real-world diagnostic 
            
102
scenarios. Lastly, existing evaluation methods lack comprehensive 
102
scenarios. Lastly, existing evaluation methods lack comprehensive 
            
103
evaluations of end-to-end real clinical scenarios. These limitations 
103
evaluations of end-to-end real clinical scenarios. These limitations 
            
104
in benchmarks in turn obstruct advancements of LLMs and agents for 
104
in benchmarks in turn obstruct advancements of LLMs and agents for 
            
105
medicine. To address these limitations, we introduce ClinicalLab, a 
105
medicine. To address these limitations, we introduce ClinicalLab, a 
            
106
comprehensive clinical diagnosis agent alignment suite, including 
106
comprehensive clinical diagnosis agent alignment suite, including 
            
107
ClinicalBench, ClinicalMetrics, and ClinicalAgent, to promote 
107
ClinicalBench, ClinicalMetrics, and ClinicalAgent, to promote 
            
108
development of clinical diagnostic agents.",
108
development of clinical diagnostic agents.",
            
n
109
  "num_resources": 0,
n
109
  "num_resources": 1,
            
110
  "num_tags": 4,
110
  "num_tags": 4,
            
111
  "organization": {
111
  "organization": {
            
112
    "approval_status": "approved",
112
    "approval_status": "approved",
            
113
    "created": "2024-11-25T12:11:38.292601",
113
    "created": "2024-11-25T12:11:38.292601",
            
114
    "description": "",
114
    "description": "",
            
115
    "id": "079d46db-32df-4b48-91f3-0a8bc8f69559",
115
    "id": "079d46db-32df-4b48-91f3-0a8bc8f69559",
            
116
    "image_url": "",
116
    "image_url": "",
            
117
    "is_organization": true,
117
    "is_organization": true,
            
118
    "name": "no-organization",
118
    "name": "no-organization",
            
119
    "state": "active",
119
    "state": "active",
            
120
    "title": "No Organization",
120
    "title": "No Organization",
            
121
    "type": "organization"
121
    "type": "organization"
            
122
  },
122
  },
            
123
  "owner_org": "079d46db-32df-4b48-91f3-0a8bc8f69559",
123
  "owner_org": "079d46db-32df-4b48-91f3-0a8bc8f69559",
            
124
  "private": false,
124
  "private": false,
            
125
  "relationships_as_object": [],
125
  "relationships_as_object": [],
            
126
  "relationships_as_subject": [],
126
  "relationships_as_subject": [],
            
t
127
  "resources": [],
t
127
  "resources": [
            
128
    {
            
129
      "cache_last_updated": null,
            
130
      "cache_url": null,
            
131
      "created": "2024-12-03T10:49:30",
            
132
      "data": [
            
133
        "dcterms:title",
            
134
        "dcterms:accessRights",
            
135
        "dcterms:creator",
            
136
        "dcterms:description",
            
137
        "dcterms:issued",
            
138
        "dcterms:language",
            
139
        "dcterms:identifier",
            
140
        "dcat:theme",
            
141
        "dcterms:type",
            
142
        "dcat:keyword",
            
143
        "dcat:landingPage",
            
144
        "dcterms:hasVersion",
            
145
        "dcterms:format",
            
146
        "mls:task",
            
147
        "datacite:isDescribedBy"
            
148
      ],
            
149
      "description": "The json representation of the dataset with its 
            
150
distributions based on DCAT.",
            
151
      "format": "JSON",
            
152
      "hash": "",
            
153
      "id": "45bd62bb-2707-47a3-b71b-c3a38396ce59",
            
154
      "last_modified": "2024-12-03T10:02:05.173526",
            
155
      "metadata_modified": "2024-12-03T10:02:05.183951",
            
156
      "mimetype": "application/json",
            
157
      "mimetype_inner": null,
            
158
      "name": "Original Metadata",
            
159
      "package_id": "8a350f8a-13df-4d14-9124-09ebe1867998",
            
160
      "position": 0,
            
161
      "resource_type": null,
            
162
      "size": 2035,
            
163
      "state": "active",
            
164
      "url": 
            
165
resource/45bd62bb-2707-47a3-b71b-c3a38396ce59/download/metadata.json",
            
166
      "url_type": "upload"
            
167
    }
            
168
  ],
            
128
  "services_used_list": "",
169
  "services_used_list": "",
            
129
  "state": "active",
170
  "state": "active",
            
130
  "tags": [
171
  "tags": [
            
131
    {
172
    {
            
132
      "display_name": "Clinical Agent",
173
      "display_name": "Clinical Agent",
            
133
      "id": "d995d8ab-a864-454d-9f60-3f0ba596ddb3",
174
      "id": "d995d8ab-a864-454d-9f60-3f0ba596ddb3",
            
134
      "name": "Clinical Agent",
175
      "name": "Clinical Agent",
            
135
      "state": "active",
176
      "state": "active",
            
136
      "vocabulary_id": null
177
      "vocabulary_id": null
            
137
    },
178
    },
            
138
    {
179
    {
            
139
      "display_name": "Clinical Diagnosis",
180
      "display_name": "Clinical Diagnosis",
            
140
      "id": "dffff827-b706-41a7-9626-524ff59ec3ca",
181
      "id": "dffff827-b706-41a7-9626-524ff59ec3ca",
            
141
      "name": "Clinical Diagnosis",
182
      "name": "Clinical Diagnosis",
            
142
      "state": "active",
183
      "state": "active",
            
143
      "vocabulary_id": null
184
      "vocabulary_id": null
            
144
    },
185
    },
            
145
    {
186
    {
            
146
      "display_name": "Medical Imaging",
187
      "display_name": "Medical Imaging",
            
147
      "id": "10261ea5-4420-472c-8004-e29d20a88fb8",
188
      "id": "10261ea5-4420-472c-8004-e29d20a88fb8",
            
148
      "name": "Medical Imaging",
189
      "name": "Medical Imaging",
            
149
      "state": "active",
190
      "state": "active",
            
150
      "vocabulary_id": null
191
      "vocabulary_id": null
            
151
    },
192
    },
            
152
    {
193
    {
            
153
      "display_name": "Natural Language Processing",
194
      "display_name": "Natural Language Processing",
            
154
      "id": "035ca6e6-7473-43c7-82b3-cb1c2b4131fe",
195
      "id": "035ca6e6-7473-43c7-82b3-cb1c2b4131fe",
            
155
      "name": "Natural Language Processing",
196
      "name": "Natural Language Processing",
            
156
      "state": "active",
197
      "state": "active",
            
157
      "vocabulary_id": null
198
      "vocabulary_id": null
            
158
    }
199
    }
            
159
  ],
200
  ],
            
160
  "title": "ClinicalLab: A Comprehensive Clinical Diagnosis Agent 
201
  "title": "ClinicalLab: A Comprehensive Clinical Diagnosis Agent 
            
161
Alignment Suite",
202
Alignment Suite",
            
162
  "type": "dataset",
203
  "type": "dataset",
            
163
  "version": ""
204
  "version": ""
            
164
}
205
}