🚔 feat: OpenAI Moderation update

api/cache/getLogStores.js

@@ -79,6 +79,7 @@ const namespaces = {
   [ViolationTypes.ILLEGAL_MODEL_REQUEST]: createViolationInstance(
     ViolationTypes.ILLEGAL_MODEL_REQUEST,
   ),
+  [ViolationTypes.MODERATION]: createViolationInstance(ViolationTypes.MODERATION),
   logins: createViolationInstance('logins'),
   [CacheKeys.ABORT_KEYS]: abortKeys,
   [CacheKeys.TOKEN_CONFIG]: tokenConfig,

api/server/middleware/moderateText.js

@@ -1,41 +1,148 @@
-const axios = require('axios');
-const { ErrorTypes } = require('librechat-data-provider');
+const OpenAI = require('openai');
+const { ErrorTypes, ViolationTypes } = require('librechat-data-provider');
+const { getCustomConfig } = require('~/server/services/Config');
+const { isEnabled } = require('~/server/utils');
 const denyRequest = require('./denyRequest');
+const { logViolation } = require('~/cache');
 const { logger } = require('~/config');
 
+const DEFAULT_ACTIONS = Object.freeze({
+  violation: 2,
+  blockMessage: true,
+  log: true,
+});
+
+// Pre-compile threshold map for faster lookups
+const DEFAULT_THRESHOLDS = new Map();
+
+function formatViolation(violation) {
+  return {
+    category: violation.category,
+    score: Math.round(violation.score * 100) / 100,
+    threshold: violation.threshold,
+    severity: getSeverityLevel(violation.score),
+  };
+}
+
+function getSeverityLevel(score) {
+  if (score >= 0.9) {
+    return 'HIGH';
+  }
+  if (score >= 0.7) {
+    return 'MEDIUM';
+  }
+  return 'LOW';
+}
+
+function formatViolationsLog(violations, userId = 'unknown') {
+  const violationsStr = violations
+    .map((v) => `${v.category}:${v.score}>${v.threshold}`)
+    .join(' | ');
+
+  return `userId=${userId} violations=[${violationsStr}]`;
+}
+
 async function moderateText(req, res, next) {
-  if (process.env.OPENAI_MODERATION === 'true') {
-    try {
-      const { text } = req.body;
-
-      const response = await axios.post(
-        process.env.OPENAI_MODERATION_REVERSE_PROXY || 'https://api.openai.com/v1/moderations',
-        {
-          input: text,
-        },
-        {
-          headers: {
-            'Content-Type': 'application/json',
-            Authorization: `Bearer ${process.env.OPENAI_MODERATION_API_KEY}`,
-          },
-        },
+  if (!isEnabled(process.env.OPENAI_MODERATION)) {
+    return next();
+  }
+
+  const moderationKey = process.env.OPENAI_MODERATION_API_KEY;
+  if (!moderationKey) {
+    logger.error('Missing OpenAI moderation API key');
+    return denyRequest(req, res, { message: 'Moderation configuration error' });
+  }
+
+  const { text } = req.body;
+  if (!text?.length || typeof text !== 'string') {
+    return denyRequest(req, res, { type: ErrorTypes.VALIDATION, message: 'Invalid text input' });
+  }
+
+  try {
+    const customConfig = await getCustomConfig();
+
+    if (!moderateText.openai) {
+      moderateText.openai = new OpenAI({ apiKey: moderationKey });
+    }
+
+    const response = await moderateText.openai.moderations.create({
+      model: 'omni-moderation-latest',
+      input: text,
+    });
+
+    if (!Array.isArray(response.results)) {
+      throw new Error('Invalid moderation API response format');
+    }
+
+    const violations = checkViolations(response.results, customConfig).map(formatViolation);
+
+    if (violations.length === 0) {
+      return next();
+    }
+
+    const actions = Object.assign({}, DEFAULT_ACTIONS, customConfig?.moderation?.actions);
+
+    if (actions.log) {
+      const userId = req.user?.id || 'anonymous';
+      logger.warn(
+        '[moderateText] Content moderation violations: ' + formatViolationsLog(violations, userId),
       );
+    }
+
+    if (!actions.blockMessage) {
+      return next();
+    }
+
+    if (actions.violation > 0) {
+      logViolation(req, res, ViolationTypes.MODERATION, { violations }, actions.violation);
+    }
+
+    return denyRequest(req, res, {
+      type: ErrorTypes.MODERATION,
+      message: `Content violates moderation policies: ${violations
+        .map((v) => v.category)
+        .join(', ')}`,
+      violations: violations,
+    });
+  } catch (error) {
+    const errorDetails =
+      process.env.NODE_ENV === 'production'
+        ? { message: error.message }
+        : {
+          error: error.message,
+          stack: error.stack,
+          status: error.response?.status,
+        };
+
+    logger.error('Moderation error:', errorDetails);
+
+    return denyRequest(req, res, {
+      type: ErrorTypes.MODERATION,
+      message: 'Content moderation check failed',
+    });
+  }
+}
+
+function checkViolations(results, customConfig) {
+  const violations = [];
+  const categories = customConfig?.moderation?.categories || {};
+
+  for (const result of results) {
+    for (const [category, score] of Object.entries(result.category_scores)) {
+      const categoryConfig = categories[category];
+
+      if (categoryConfig?.enabled === false) {
+        continue;
+      }
 
-      const results = response.data.results;
-      const flagged = results.some((result) => result.flagged);
+      const threshold = categoryConfig?.threshold || DEFAULT_THRESHOLDS.get(category) || 0.7;
 
-      if (flagged) {
-        const type = ErrorTypes.MODERATION;
-        const errorMessage = { type };
-        return await denyRequest(req, res, errorMessage);
+      if (score >= threshold) {
+        violations.push({ category, score, threshold });
       }
-    } catch (error) {
-      logger.error('Error in moderateText:', error);
-      const errorMessage = 'error in moderation check';
-      return await denyRequest(req, res, errorMessage);
     }
   }
-  next();
+  return violations;
 }
 
 module.exports = moderateText;

api/server/routes/ask/gptPlugins.js

@@ -21,8 +21,8 @@ const { logger } = require('~/config');
 
 const router = express.Router();
 
-router.use(moderateText);
 router.post('/abort', handleAbort());
+router.use(moderateText);
 
 router.post(
   '/',

api/server/routes/ask/openAI.js

@@ -11,8 +11,8 @@ const {
 } = require('~/server/middleware');
 
 const router = express.Router();
-router.use(moderateText);
 router.post('/abort', handleAbort());
+router.use(moderateText);
 
 router.post(
   '/',

api/server/routes/edit/gptPlugins.js

@@ -20,8 +20,8 @@ const { logger } = require('~/config');
 
 const router = express.Router();
 
-router.use(moderateText);
 router.post('/abort', handleAbort());
+router.use(moderateText);
 
 router.post(
   '/',

api/server/routes/edit/openAI.js

@@ -11,8 +11,8 @@ const {
 } = require('~/server/middleware');
 
 const router = express.Router();
-router.use(moderateText);
 router.post('/abort', handleAbort());
+router.use(moderateText);
 
 router.post(
   '/',