diff --git a/src/lib/components/Doomers.svelte b/src/lib/components/Doomers.svelte
index 3bc70010..65249527 100644
--- a/src/lib/components/Doomers.svelte
+++ b/src/lib/components/Doomers.svelte
@@ -145,6 +145,13 @@
number: 0.8,
source: 'https://twitter.com/DanHendrycks/status/1642394635657162753'
},
+ {
+ name: 'Roman Yampolskiy',
+ title: 'AI Safety Researcher',
+ probability: '98%',
+ number: 0.02,
+ source: 'https://www.scientificamerican.com/article/ai-safety-research-only-enables-the-dangers-of-runaway-superintelligence/#:~:text=1%20percent%20chance%20to%20maybe%202%20percent.'
+ },
{
name: 'Eliezer Yudkowsky',
title: 'Founder of MIRI',
diff --git a/src/posts/learn.md b/src/posts/learn.md
index 39ff718b..cfdc5ca5 100644
--- a/src/posts/learn.md
+++ b/src/posts/learn.md
@@ -47,6 +47,7 @@ Here are some resources to get you started.
- [Human Compatible: Artificial Intelligence and the Problem of Control](https://www.goodreads.com/en/book/show/44767248) (Stuart Russell)
- [Our Final Invention: Artificial Intelligence and the End of the Human Era](https://www.goodreads.com/en/book/show/17286699) (James Barrat)
- [The Precipice: Existential Risk and the Future of Humanity](https://www.goodreads.com/en/book/show/50963653) (Toby Ord)
+- [Uncontrollable](https://www.goodreads.com/book/show/202416160-uncontrollable) (Darren McKee)
## Courses
diff --git a/src/posts/polls-and-surveys.md b/src/posts/polls-and-surveys.md
index f4932dc4..45c63392 100644
--- a/src/posts/polls-and-surveys.md
+++ b/src/posts/polls-and-surveys.md
@@ -8,7 +8,7 @@ How much do regular people and experts worry about AI risks?
## Catastrophic risks from AI
- **[AI researchers, AIImpacts](https://aiimpacts.org/2022-expert-survey-on-progress-in-ai/)**: give "really bad outcomes (such as human extinction)" a 14% probability, with a median of 5%. Only 18% believe the control problem is not important.
-- **[AI engineers / startup founders, State of AI Engineering](https://elemental-croissant-32a.notion.site/State-of-AI-Engineering-2023-20c09dc1767f45988ee1f479b4a84135#694f89e86f9148cb855220ec05e9c631)**: over 60% have a [p(doom)](/pdoom) > 25%. Only 12% have a p(doom) = 0.
+- **[AI engineers/startup founders, State of AI Engineering](https://elemental-croissant-32a.notion.site/State-of-AI-Engineering-2023-20c09dc1767f45988ee1f479b4a84135#694f89e86f9148cb855220ec05e9c631)**: over 60% have a [p(doom)](/pdoom) > 25%. Only 12% have a p(doom) = 0.
- **[AI safety researchers, AlignmentForum](https://web.archive.org/web/20221013014859/https://www.alignmentforum.org/posts/QvwSr5LsxyDeaPK5s/existential-risk-from-ai-survey-results)**: respondents assigned a median probability of 20% to x-risk caused due to a lack of enough technical research, and 30% to x-risk caused due to a failure of AI systems to do what the people deploying them intended, with huge variation (for example, there are data points at both ~1% and ~99%).
- **[UK citizens, PublicFirst](https://publicfirst.co.uk/ai/)**: think there's a 9% probability humans will go extinct because of AI. About 50% of say they're very or somewhat worried about this.
- **[German citizens, Kira](https://www.zeit.de/digital/2023-04/ki-risiken-angst-umfrage-forschung-kira)**: Only 14% believe AI will have a positive influence on the world, 40% mixed, 40% negative.
@@ -20,6 +20,7 @@ How much do regular people and experts worry about AI risks?
- [**US citizens, YouGov**](https://www.vox.com/future-perfect/2023/8/18/23836362/ai-slow-down-poll-regulation): 72% want AI to slow down, 8% want to speed up. 83% of voters believe AI could accidentally cause a catastrophic event
- [**US citizens, YouGov**](https://theaipi.org/poll-shows-voters-oppose-open-sourcing-ai-models-support-regulatory-representation-on-boards-and-say-ai-risks-outweigh-benefits-2/): 73% believe AI companies should be held liable for harms from technology they create, 67% think the AI models’ power should be restricted, and 65% believe keeping AI out of the hands of bad actors is more important than providing AI’s benefits to everyone.
- [**US citizens, AIPI**](https://www.politico.com/newsletters/digital-future-daily/2023/11/29/exclusive-what-people-actually-think-about-ai-00129147): 49:20 support "an international treaty to ban any ‘smarter-than-human’ artificial intelligence (AI)?", 70:14 support "Preventing AI from quickly reaching superhuman capabilities"
+- [**US citizens, AIPI**](https://acrobat.adobe.com/id/urn:aaid:sc:VA6C2:181573f9-b3ce-43c0-b106-1d06558008e6): 77% In favor of going slow with AI development, 8% want to speed up. 56% Agree to stop/slow down vs 27% disagree. 50% Feel afraid of AI.
- [**US CS professors, Axios Generation Lab**](https://www.axios.com/2023/09/05/ai-regulations-expert-survey): About 1 in 5 predicted AI will "definitely" stay in human control. The rest were split between those saying AI will "probably" or "definitely" get out of human control and those saying "probably not".
Just 1 in 6 said AI shouldn't or can't be regulated. Only a handful trust the private sector to self-regulate.
- [**US citizens, Sentience Institute**](https://www.sentienceinstitute.org/aims-survey-supplement-2023): There was broad support for steps that could be taken to slow down development. People supported public campaigns to slow down AI development (71.3%), government regulation that slows down development (71.0%), and a six-month pause on some kinds of AI developments (69.1%). Support for a ban on artificial general intelligence (AGI) that is smarter than humans was 62.9%.
diff --git a/src/routes/api/posts/+server.ts b/src/routes/api/posts/+server.ts
index 1b20b707..8f96acc8 100644
--- a/src/routes/api/posts/+server.ts
+++ b/src/routes/api/posts/+server.ts
@@ -5,9 +5,17 @@ import { communitiesMeta } from '../../communities/communities'
import { meta as pdoomMeta } from '../../pdoom/meta'
import { meta as quotesMeta } from '../../quotes/meta'
import { meta as emailBuilderMeta } from '../../email-builder/meta'
+import { meta as scorecardMeta } from '../../scorecard/meta'
/** When adding an extra route, make sure to add the metadata here for SEO purposes */
-const hardCodedPages: Post[] = [outcomesMeta, communitiesMeta, pdoomMeta, quotesMeta, emailBuilderMeta]
+const hardCodedPages: Post[] = [
+ outcomesMeta,
+ communitiesMeta,
+ pdoomMeta,
+ quotesMeta,
+ emailBuilderMeta,
+ scorecardMeta
+]
async function getPosts() {
let posts: Post[] = []
diff --git a/src/routes/scorecard/+page.svelte b/src/routes/scorecard/+page.svelte
new file mode 100644
index 00000000..5e44aca0
--- /dev/null
+++ b/src/routes/scorecard/+page.svelte
@@ -0,0 +1,26 @@
+
+
+
+
+
{title}
+
+ AI Companies are building some of the most impactful technology in the world. PauseAI keeps track
+ of how responsible these companies are acting, and what they are doing to prevent various risks.
+
+
+ This scorecard is a work in progress. The scores and explanations in this PR are temporary
+ values, and do not yet represent PauseAI's or any jury's opinions. Please contribute suggestion on github!
+
+
+
diff --git a/src/routes/scorecard/categories.ts b/src/routes/scorecard/categories.ts
new file mode 100644
index 00000000..4890c6b5
--- /dev/null
+++ b/src/routes/scorecard/categories.ts
@@ -0,0 +1,62 @@
+import type { Category } from './types'
+
+export const categories: Category[] = [
+ {
+ name: 'Acknowledge',
+ key: 'acknowledge',
+ explanation:
+ // 10: Pushing the conversation forward, making it more mainstream
+ // 8: Has devoted substatial resources to public communication of AI risks
+ // 7: Has made an attemt to communicate concerns to the public
+ // 6: Has acknowledged xrisk, but no attempt to highlight these concerns
+ // 5: E.g.no acknowledgement of risks, but also no denial.
+ // 4: Active downlplaying of concerns, but in reasonable way
+ // 3: Minimization of risk, dismissal, but not explicitly denying or attempt to move the conversation
+ // 0: Actively denying the risks of AI, not discussing in good faith, attacking advocates on a personal level
+ // How likely are they saying the risks are to happen?
+ 'How well does the company acknowledge the risks of AI? Do they acknowledge all the risks? How long did it take them to acknowledge the risks?'
+ },
+ {
+ name: 'Lobby',
+ key: 'lobby',
+ // 10: Actively introducing draft legislations / concrete proposals that harm their internal business model but that are good for AI safety
+ // 5: No lobbying, no attempt to influence regulation in any direction
+ // 1: Using personal connections, money or financial incentives to influence regulation in a way that is very harmful to AI safety.
+ // 0: Criminal lobbying, using illegal means to prevent meaningful AI safety regulation.
+ explanation:
+ 'How much does the company lobby for AI regulation? Do they lobby for deregulation or for stricter regulation?'
+ },
+ {
+ name: 'Deployment',
+ key: 'deployment',
+ explanation:
+ // What principles drive your decisions about what (not) to deploy?
+ // To what degree do you control access to your technology?
+ // And do you monitor usage?
+ // Are you willing to introduce new capabilities that others have not?
+ // What are you open sourcing?
+ // What are your plans
+ // 0: Releases model weights of frontier models regardless of the risks.
+ // 2: Releases frontier model weights, but only after testing for safety.
+ // 3: Open model weights for models that are behind the state of the art, where comparable models have demonstrated to be (catastrophically) safe.
+ // 5: No open source,
+ 'How long does the company wait before deploying a new model? Do they wait until they have a good understanding of the risks of the model? Are their models tested thoroughly?'
+ },
+ {
+ name: 'Research',
+ key: 'research',
+ explanation:
+ // 10:
+ // 5:
+ // 0:
+ 'Is the company doing research on AI safety or alignment? Are they doing research on how to mitigate the risks of AI?'
+ }
+ // Frontier column
+ // Are you pushing capabilities forward or are you following others to provide services?
+ // This is where OpenAI scores 0 points.
+ // To what extent do their future plans advance the state of the art?
+
+ // Culture column
+ // To what extent do you have a culture of safety?
+ // Do the employees have a good understanding of the risks of AI?
+]
diff --git a/src/routes/scorecard/companies.ts b/src/routes/scorecard/companies.ts
new file mode 100644
index 00000000..f5a2de87
--- /dev/null
+++ b/src/routes/scorecard/companies.ts
@@ -0,0 +1,188 @@
+import type { Company } from './types'
+
+const companiesSource: Company[] = [
+ {
+ name: 'OpenAI',
+ acknowledge: {
+ explanation:
+ "Has now publicly acknowledged most of the AI risks, including existential risk. However, it took them many years to do so. Sam Altman wasn't honest about his 'worst nightmare' during the Senate hearing in May 2023.",
+ score: 7
+ },
+ lobby: {
+ explanation:
+ 'OpenAI has publicly called for stricter regulation, but also tried to severely weaken how the EU AI Act regulates frontier models. Sam Altman went on a world tour where he emphasized the need for an international agency to regulate large training runs.',
+ score: 4
+ },
+ deployment: {
+ explanation:
+ 'OpenAI releasing ChatGPT to the public was an absolute shockwave. However, GPT-4 was released 7 months after it finished training, during which they did a lot of safety work like red-teaming by ARC.',
+ score: 6
+ },
+ research: {
+ explanation:
+ 'OpenAI has published a lot of impactful AI safety research and has dedicated a substantial amount of resources to their "superalignement" project.',
+ score: 7
+ }
+ },
+ {
+ name: 'Google DeepMind',
+ acknowledge: {
+ explanation:
+ // Hassabis calls for "Cern for AI Safety" https://www.theguardian.com/technology/2023/oct/24/ai-risk-climate-crisis-google-deepmind-chief-demis-hassabis-regulation
+ "Hassabis has publicly acknowledged the existential risk from AI. They weren't as explicit about this as OpenAI.",
+ score: 6
+ },
+ lobby: {
+ // https://corporateeurope.org/en/2023/11/byte-byte
+ explanation:
+ 'They have lobbied to shift the burden of responsibility onto users of AI instead of the ones building the AI.',
+ score: 3
+ },
+ deployment: {
+ explanation:
+ 'Google used to be very careful with releasing models, but that changed in 2023. When releasing Palm 2 in 2023, Google skipped any mention of safety in their release paper. However, with the release of Gemini in december 2023, they have written more extensively on this.',
+ score: 5
+ },
+ research: {
+ explanation: 'Google DeepMind has published quite a few impactful AI safety papers',
+ score: 5
+ }
+ },
+ {
+ name: 'Microsoft',
+ acknowledge: {
+ explanation:
+ // Satya isn't that worried about election interference: https://www.verdict.co.uk/microsoft-ceo-satya-nadella-is-sanguine-about-ai-risk-to-elections/?cf-view
+ // Xrisk not true: https://www.euronews.com/my-europe/2023/06/29/microsoft-chief-says-ai-is-not-an-existential-risk-to-mankind-but-human-oversight-needed
+ // Does acknowledge biases, job displacement: https://www.verdict.co.uk/microsoft-ceo-satya-nadella-is-sanguine-about-ai-risk-to-elections/?cf-view
+ 'Satya Nadella does not acknowledge the existential risk from AI, nor does he seem worried about election interference.',
+ score: 2
+ },
+ lobby: {
+ explanation:
+ // https://corporateeurope.org/en/2023/11/byte-byte
+ 'Microsoft has lobbied to shift the burden of responsibility onto users of AI instead of the ones building the AI.',
+ score: 2
+ },
+ deployment: {
+ explanation:
+ // Never released open source models.
+ "Microsoft released an unfinished, sometimes even unhinged Bing (based on OpenAI's GPT-4) in April 2023. It was embarrassing and dangerous. OpenAI urged Microsoft not to do this - they did it anyway.",
+ score: 4
+ },
+ research: {
+ explanation:
+ // https://www.theverge.com/2023/3/13/23638823/microsoft-ethics-society-team-responsible-ai-layoffs
+ "Microsoft has published almost no safety research and recently laid off their 'ethics and society' team.",
+ score: 1
+ }
+ },
+ {
+ name: 'Meta',
+ acknowledge: {
+ explanation:
+ "Meta's chief scientist Yann LeCun is one of the most notorious AI risk deniers, one of the loudest voices in the field. On Twitter he often resorts to ad hominem attacks and refuses to engage honestly.",
+ score: 0
+ },
+ lobby: {
+ explanation:
+ "Meta doesn't seem to be as active lobbying politicians through backchannels as other companies. They are quite publicly calling for less regulation, though.",
+ score: 3
+ },
+ deployment: {
+ explanation:
+ 'Meta has leaked and released the weights of powerful AI models. They get some points for improving how much they worked on safety in their latest LLAMA 2 release.',
+ score: 2
+ },
+ research: {
+ explanation:
+ 'Meta has published almost no safety research. They have a few papers on adversarial examples, but nothing on existential risk.',
+ score: 1
+ }
+ },
+ {
+ name: 'Anthropic',
+ acknowledge: {
+ explanation:
+ 'Anthropic has publicly acknowledged and brought attention to many AI risks, including the existential risk. Their CEO Dario Amodei has been one of the most vocal proponents of AI safety.',
+ score: 9
+ },
+ lobby: {
+ // https://pitchbook.com/news/articles/generative-AI-Capitol-Hill-VC
+ explanation:
+ "Anthropic has spent (a little) money lobbying, but it's unclear what they are pushing for.",
+ score: 5
+ },
+ deployment: {
+ explanation:
+ 'Anthropic seems to have a policy of not deploying SOTA models. Anthropic sat on Claude - and waited with deploying it until ChatGPT came out. However, Anthropic was very optimistic about Claude 2 being "unjailbreakable", which was disproved in minutes after releasing the model.',
+ score: 7
+ },
+ research: {
+ explanation:
+ 'Anthropic has published very important advancements in AI safety research, especially in the field of interpretability.',
+ score: 9
+ }
+ },
+ {
+ name: 'Inflection',
+ acknowledge: {
+ // https://forum.effectivealtruism.org/posts/JsjQRqvRc5pFmeSoj/what-do-we-know-about-mustafa-suleyman-s-position-on-ai
+ // Calls existential risk a "completely bonkers distraction"
+ explanation:
+ 'Their CEO Mustafa Suleyman has written extensively about many AI risks in his book. However, he calls existential risks a "completely bonkers distraction".',
+ score: 5
+ },
+ lobby: {
+ explanation: 'No lobbying as far as we know of yet.',
+ score: 5
+ },
+ deployment: {
+ explanation: 'Not sure tbd',
+ score: 4
+ },
+ research: {
+ explanation: 'No AI safety research published by Inflection.',
+ score: 0
+ }
+ },
+ {
+ name: 'Mistral',
+ acknowledge: {
+ // Thinks bio risks are hypothetical, not to be concerned about : https://youtu.be/EMOFRDOMIiU?si=yN2xDshaaEMfr9mQ&t=1186
+ // Say guardrails on both output and input side are needed.
+ // Says users should be responsible for safety.
+ // Says x-risk is "philosophical", we're not at that stage, it's very abstract. https://youtu.be/EMOFRDOMIiU?si=Zk0EUuRv90iIaFeX&t=1572
+ explanation: 'They have not yet publicly acknowledged any AI risks. ',
+ score: 3
+ },
+ lobby: {
+ explanation:
+ 'Used an ex-minister to actively lobby against the EU AI Act, almost causing it to not include frontier model regulations.',
+ score: 0
+ },
+ deployment: {
+ explanation: 'They released their models without any safety work as torrents.',
+ score: 0
+ },
+ research: {
+ explanation: 'No AI safety research published by Mystral.',
+ score: 0
+ }
+ }
+]
+
+export const companies: Company[] = companiesSource
+ .map((company) => {
+ const { name, acknowledge, lobby, deployment, research } = company
+ const totalScore = (acknowledge.score + lobby.score + deployment.score + research.score) / 4
+ return {
+ name,
+ acknowledge,
+ lobby,
+ deployment,
+ research,
+ totalScore
+ }
+ })
+ .sort((a, b) => b.totalScore - a.totalScore)
diff --git a/src/routes/scorecard/meta.ts b/src/routes/scorecard/meta.ts
new file mode 100644
index 00000000..20ed0ad4
--- /dev/null
+++ b/src/routes/scorecard/meta.ts
@@ -0,0 +1,10 @@
+import type { Post } from '$lib/types'
+
+export const meta: Post = {
+ title: 'AI Company Safety Scorecard',
+ description:
+ 'How much are AI companies doing to prevent disaster? This scorecard is a work in progress.',
+ date: '2024-01-04',
+ slug: 'scorecard',
+ categories: []
+}
diff --git a/src/routes/scorecard/types.ts b/src/routes/scorecard/types.ts
new file mode 100644
index 00000000..218bf254
--- /dev/null
+++ b/src/routes/scorecard/types.ts
@@ -0,0 +1,23 @@
+export type Company = {
+ name: string
+ title?: string
+ acknowledge: Score
+ lobby: Score
+ deployment: Score
+ research: Score
+ totalScore?: number
+}
+
+export type Score = {
+ explanation: string
+ /** 0 - 10*/
+ score: number
+}
+
+export type catagoryTypes = 'acknowledge' | 'lobby' | 'deployment' | 'research'
+
+export type Category = {
+ name: string
+ key: catagoryTypes
+ explanation: string
+}