Skip to content

Commit 87b2e12

Browse files
joepiojoepio
committed
Better scorecard
1 parent 7da7872 commit 87b2e12

File tree

3 files changed

+46
-16
lines changed

3 files changed

+46
-16
lines changed

‎src/routes/scorecard/+page.svelte‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@
1717
</p>
1818
<p>
1919
<em
20-
>This scorecard is a work in progress. Please <a
20+
>This scorecard is a work in progress. The scores and explanations in this PR are temporary
21+
values, and do not yet represent PauseAI's or any jury's opinions. Please <a
2122
href="https://github.com/joepio/pauseai/pull/32">contribute suggestion on github</a
2223
>!</em
2324
>

‎src/routes/scorecard/Scorecard.svelte‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
{#each categories as category (category.name)}
1919
<Cell title={category.name} explanation={category.explanation} {showExplanation} />
2020
{/each}
21-
<Cell title="Total" explanation="Sum of the scores" {showExplanation} />
21+
<Cell title="Total" explanation="Average of the scores" {showExplanation} />
2222
</tr>
2323
</thead>
2424
<tbody>
@@ -36,7 +36,8 @@
3636
/>
3737
{/if}
3838
{/each}
39-
<td class="total">{company.totalScore}</td>
39+
<Cell score={company.totalScore} explanation="Average of all scores." {showExplanation} />
40+
<!-- <td class="total">{company.totalScore}</td> -->
4041
</tr>
4142
{/each}
4243
</tbody>

‎src/routes/scorecard/companies.ts‎

Lines changed: 41 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,25 +10,26 @@ const companiesSource: Company[] = [
1010
},
1111
lobby: {
1212
explanation:
13-
'OpenAI has publicly called for stricter regulation, but also tried to severely weaken how the EU AI Act regulates frontier models.',
13+
'OpenAI has publicly called for stricter regulation, but also tried to severely weaken how the EU AI Act regulates frontier models. Sam Altman went on a world tour where he emphasized the need for an international agency to regulate large training runs.',
1414
score: 4
1515
},
1616
deployment: {
1717
explanation:
18-
'GPT-4 was released 7 months after it finished training, during which they did a lot of safety work like red-teaming by ARC.',
19-
score: 8
18+
'OpenAI releasing ChatGPT to the public was an absolute shockwave. However, GPT-4 was released 7 months after it finished training, during which they did a lot of safety work like red-teaming by ARC.',
19+
score: 6
2020
},
2121
research: {
2222
explanation:
2323
'OpenAI has published a lot of impactful AI safety research and has dedicated a substantial amount of resources to their "superalignement" project.',
24-
score: 8
24+
score: 7
2525
}
2626
},
2727
{
2828
name: 'Google DeepMind',
2929
acknowledge: {
3030
explanation:
31-
"Hassabis has now publicly acknowledged the existential risk from AI. They weren't as quick with this as OpenAI",
31+
// Hassabis calls for "Cern for AI Safety" https://www.theguardian.com/technology/2023/oct/24/ai-risk-climate-crisis-google-deepmind-chief-demis-hassabis-regulation
32+
"Hassabis has publicly acknowledged the existential risk from AI. They weren't as explicit about this as OpenAI.",
3233
score: 6
3334
},
3435
lobby: {
@@ -51,8 +52,10 @@ const companiesSource: Company[] = [
5152
name: 'Microsoft',
5253
acknowledge: {
5354
explanation:
54-
// https://www.euronews.com/my-europe/2023/06/29/microsoft-chief-says-ai-is-not-an-existential-risk-to-mankind-but-human-oversight-needed
55-
'Microsoft does not acknowledge the existential risk from AI.',
55+
// Satya isn't that worried about election interference: https://www.verdict.co.uk/microsoft-ceo-satya-nadella-is-sanguine-about-ai-risk-to-elections/?cf-view
56+
// Xrisk not true: https://www.euronews.com/my-europe/2023/06/29/microsoft-chief-says-ai-is-not-an-existential-risk-to-mankind-but-human-oversight-needed
57+
// Does acknowledge biases, job displacement: https://www.verdict.co.uk/microsoft-ceo-satya-nadella-is-sanguine-about-ai-risk-to-elections/?cf-view
58+
'Satya Nadella does not acknowledge the existential risk from AI, nor does he seem worried about election interference.',
5659
score: 2
5760
},
5861
lobby: {
@@ -77,7 +80,7 @@ const companiesSource: Company[] = [
7780
name: 'Meta',
7881
acknowledge: {
7982
explanation:
80-
"Meta's chief scientist Yann LeCun is one of the most notories AI risk deniers, one of the loudest voices in the field. On Twitter he often resorts to ad hominem attacks and refuses to engage honestly.",
83+
"Meta's chief scientist Yann LeCun is one of the most notorious AI risk deniers, one of the loudest voices in the field. On Twitter he often resorts to ad hominem attacks and refuses to engage honestly.",
8184
score: 0
8285
},
8386
lobby: {
@@ -111,8 +114,8 @@ const companiesSource: Company[] = [
111114
},
112115
deployment: {
113116
explanation:
114-
'Anthropic was very optimistic about Claude 2 being "unjailbreakable", which was disproved in minutes after releasing the model.',
115-
score: 5
117+
'Anthropic seems to have a policy of not deploying SOTA models. Anthropic sat on Claude - and waited with deploying it until ChatGPT came out. However, Anthropic was very optimistic about Claude 2 being "unjailbreakable", which was disproved in minutes after releasing the model.',
118+
score: 7
116119
},
117120
research: {
118121
explanation:
@@ -124,9 +127,10 @@ const companiesSource: Company[] = [
124127
name: 'Inflection',
125128
acknowledge: {
126129
// https://forum.effectivealtruism.org/posts/JsjQRqvRc5pFmeSoj/what-do-we-know-about-mustafa-suleyman-s-position-on-ai
130+
// Calls existential risk a "completely bonkers distraction"
127131
explanation:
128-
'Their CEO Mustafa Suleyman has written extensively about catastrophic AI risks in his book.',
129-
score: 8
132+
'Their CEO Mustafa Suleyman has written extensively about many AI risks in his book. However, he calls existential risks a "completely bonkers distraction".',
133+
score: 5
130134
},
131135
lobby: {
132136
explanation: 'No lobbying as far as we know of yet.',
@@ -140,13 +144,37 @@ const companiesSource: Company[] = [
140144
explanation: 'No AI safety research published by Inflection.',
141145
score: 0
142146
}
147+
},
148+
{
149+
name: 'Mistral',
150+
acknowledge: {
151+
// Thinks bio risks are hypothetical, not to be concerned about : https://youtu.be/EMOFRDOMIiU?si=yN2xDshaaEMfr9mQ&t=1186
152+
// Say guardrails on both output and input side are needed.
153+
// Says users should be responsible for safety.
154+
// Says x-risk is "philosophical", we're not at that stage, it's very abstract. https://youtu.be/EMOFRDOMIiU?si=Zk0EUuRv90iIaFeX&t=1572
155+
explanation: 'They have not yet publicly acknowledged any AI risks. ',
156+
score: 3
157+
},
158+
lobby: {
159+
explanation:
160+
'They actively lobbied against the EU AI Act, almost causing it to not include frontier model regulations.',
161+
score: 2
162+
},
163+
deployment: {
164+
explanation: 'They released their models without any safety work as torrents.',
165+
score: 0
166+
},
167+
research: {
168+
explanation: 'No AI safety research published by Mystral.',
169+
score: 0
170+
}
143171
}
144172
]
145173

146174
export const companies: Company[] = companiesSource
147175
.map((company) => {
148176
const { name, acknowledge, lobby, deployment, research } = company
149-
const totalScore = acknowledge.score + lobby.score + deployment.score + research.score
177+
const totalScore = (acknowledge.score + lobby.score + deployment.score + research.score) / 4
150178
return {
151179
name,
152180
acknowledge,

0 commit comments

Comments
 (0)