Merge branch 'master' of github.com:openforis/arena-server into feat/auto-scaling

Author: SteRiccio

.env.template

@@ -22,6 +22,18 @@ RATE_LIMIT_WINDOW_MS=900000
 RATE_LIMIT_MAX=100
 ## File upload limit in bytes (default: 1GB)
 FILE_UPLOAD_LIMIT=1073741824
+## Optional local log folder used for rolling files and S3 shipping
+LOG_FOLDER=./logs
+LOG_MAX_SIZE_BYTES=10485760
+LOG_RETENTION_DAYS=30
+LOG_UPLOAD_INTERVAL_MS=60000
+LOG_S3_PREFIX=logs
+## Optional S3-backed log shipping (requires LOG_S3_ENABLED=true and file storage S3 to be enabled)
+LOG_S3_ENABLED=false
+FILE_STORAGE_AWS_ACCESS_KEY=
+FILE_STORAGE_AWS_SECRET_ACCESS_KEY=
+FILE_STORAGE_AWS_S3_BUCKET_NAME=
+FILE_STORAGE_AWS_S3_BUCKET_REGION=
 # Security - IMPORTANT: Use strong random secrets in production (e.g., 64+ character random string)
 USER_AUTH_TOKEN_SECRET=user-auth-token-secret
 USER_2FA_SECRET=user-2fa-secret

.yarn/install-state.gz

@@ -5,6 +5,8 @@ enableScripts: true
 
 nodeLinker: node-modules
 
+npmMinimalAgeGate: 0
+
 npmPreapprovedPackages:
   - "@openforis/arena-core"
 

.yarnrc.yml

@@ -4,7 +4,7 @@
     "name": "OpenForis",
     "email": "openforis.arena@gmail.com"
   },
-  "version": "2.0.2",
+  "version": "2.0.3",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -44,6 +44,7 @@
     "typescript-eslint": "^8.59.2"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1112.0",
     "@godaddy/terminus": "^4.12.1",
     "@openforis/arena-core": "^1.5.0",
     "bcryptjs": "^3.0.3",

package.json

@@ -0,0 +1,59 @@
+import { DeleteObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'
+
+import { ProcessEnv } from '../processEnv'
+
+export type S3StorageOptions = {
+  bucketName: string
+  region: string | undefined
+  accessKeyId: string
+  secretAccessKey: string
+  endpoint?: string
+  forcePathStyle?: boolean
+}
+
+export class S3Storage {
+  static fromEnvironment(): S3Storage | null {
+    if (!ProcessEnv.fileStorageAwsEnabled) return null
+
+    return new S3Storage({
+      bucketName: ProcessEnv.fileStorageAwsS3BucketName!,
+      region: ProcessEnv.fileStorageAwsS3BucketRegion,
+      accessKeyId: ProcessEnv.fileStorageAwsAccessKey!,
+      secretAccessKey: ProcessEnv.fileStorageAwsSecretAccessKey!,
+    })
+  }
+
+  private readonly client: S3Client
+
+  constructor(private readonly options: S3StorageOptions) {
+    this.client = new S3Client({
+      region: options.region,
+      endpoint: options.endpoint,
+      forcePathStyle: options.forcePathStyle,
+      credentials: {
+        accessKeyId: options.accessKeyId,
+        secretAccessKey: options.secretAccessKey,
+      },
+    })
+  }
+
+  async putFile(key: string, body: Buffer | Uint8Array | string, contentType?: string): Promise<void> {
+    await this.client.send(
+      new PutObjectCommand({
+        Bucket: this.options.bucketName,
+        Key: key,
+        Body: body,
+        ContentType: contentType,
+      })
+    )
+  }
+
+  async deleteFile(key: string): Promise<void> {
+    await this.client.send(
+      new DeleteObjectCommand({
+        Bucket: this.options.bucketName,
+        Key: key,
+      })
+    )
+  }
+}

src/fileStorage/s3Storage.ts

@@ -0,0 +1,39 @@
+import { S3Storage } from '../s3Storage'
+
+const sendMock = jest.fn()
+
+jest.mock('@aws-sdk/client-s3', () => {
+  return {
+    S3Client: jest.fn().mockImplementation(() => ({ send: sendMock })),
+    PutObjectCommand: jest.fn().mockImplementation((params) => params),
+    DeleteObjectCommand: jest.fn().mockImplementation((params) => params),
+  }
+})
+
+describe('S3Storage', () => {
+  const originalEnv = { ...process.env }
+
+  afterEach(() => {
+    process.env = { ...originalEnv }
+    jest.clearAllMocks()
+  })
+
+  test('uploadFile sends a PutObjectCommand with the given payload and key', async () => {
+    const storage = new S3Storage({
+      bucketName: 'my-bucket',
+      region: 'eu-central-1',
+      accessKeyId: 'key',
+      secretAccessKey: 'secret',
+    })
+
+    await storage.putFile('logs/app.log', Buffer.from('hello world'), 'text/plain; charset=utf-8')
+
+    expect(sendMock).toHaveBeenCalledTimes(1)
+    expect(sendMock.mock.calls[0][0]).toMatchObject({
+      Bucket: 'my-bucket',
+      Key: 'logs/app.log',
+      ContentType: 'text/plain; charset=utf-8',
+      Body: Buffer.from('hello world'),
+    })
+  })
+})

src/fileStorage/tests/s3Storage.test.ts

@@ -36,6 +36,9 @@ export type { JobContext, JobMessageIn, JobMessageOut } from './job'
 
 export { Logger } from './log'
 
+export { S3Storage } from './fileStorage/s3Storage'
+export type { S3StorageOptions } from './fileStorage/s3Storage'
+
 export { ProcessEnv, NodeEnv } from './processEnv'
 
 export { NodeDefRepository } from './repository'

src/index.ts

@@ -1,19 +1,33 @@
+import path from 'node:path'
+
 import { configure, Logger } from 'log4js'
 
+import { ProcessEnv } from '../processEnv'
+import { startLogUploadPolling } from './logFileS3Upload'
+
 // Only display color for terminals:
 const layout = process.stdout.isTTY ? { type: 'colored' } : { type: 'basic' }
 
+startLogUploadPolling()
+
 export const getLogger = (category?: string): Logger => {
+  const fileAppender = {
+    type: 'file',
+    filename: path.join(path.resolve(ProcessEnv.logFolder), 'arena.log'),
+    maxLogSize: ProcessEnv.logMaxSizeBytes,
+    backups: 5,
+    compress: true,
+  }
+
   const log4js = configure({
     appenders: {
       console: { type: 'console', layout },
-      // { file: { type: 'file', filename: 'arena.log' }
+      file: fileAppender,
     },
     categories: {
       default: {
-        appenders: ['console'],
+        appenders: ['console', 'file'],
         level: 'debug',
-        // Appenders: ['file'], level: 'error'
       },
     },
   })

src/log/log4js.ts

@@ -0,0 +1,84 @@
+import { mkdir, readFile, readdir, rm, stat, unlink } from 'node:fs/promises'
+import path from 'node:path'
+
+import { S3Storage } from '../fileStorage/s3Storage'
+import { ProcessEnv } from '../processEnv'
+
+const withLogFolder = async (): Promise<string> => {
+  const logFolder = path.resolve(ProcessEnv.logFolder)
+  await mkdir(logFolder, { recursive: true })
+  return logFolder
+}
+
+const trimSlashes = (value: string): string => {
+  let start = 0
+  let end = value.length
+
+  while (start < end && value[start] === '/') start += 1
+  while (end > start && value[end - 1] === '/') end -= 1
+
+  return value.slice(start, end)
+}
+
+const uploadLogFileToS3 = async (logFolder: string, fileName: string, s3Storage: S3Storage): Promise<void> => {
+  const shouldDeleteAfterUpload = fileName !== 'arena.log'
+
+  const absolutePath = path.join(logFolder, fileName)
+  const fileStats = await stat(absolutePath)
+  if (shouldDeleteAfterUpload && fileStats.size === 0) return
+
+  const key = `${trimSlashes(ProcessEnv.logS3Prefix)}/${fileName}`
+  const body = await readFile(absolutePath)
+  const contentType = fileName.endsWith('.gz') ? 'application/gzip' : 'text/plain; charset=utf-8'
+
+  await s3Storage.putFile(key, body, contentType)
+
+  if (shouldDeleteAfterUpload) {
+    await unlink(absolutePath)
+  }
+}
+
+const uploadPendingLogFilesToS3 = async (logFolder: string, s3Storage: S3Storage): Promise<void> => {
+  const entries = await readdir(logFolder, { withFileTypes: true })
+
+  for (const entry of entries) {
+    if (!entry.isFile()) continue
+    await uploadLogFileToS3(logFolder, entry.name, s3Storage)
+  }
+}
+
+const cleanupStaleLogFiles = async (logFolder: string): Promise<void> => {
+  const cutoffTimestamp = Date.now() - ProcessEnv.logRetentionDays * 24 * 60 * 60 * 1000
+  const entries = await readdir(logFolder, { withFileTypes: true })
+
+  for (const entry of entries) {
+    if (!entry.isFile()) continue
+
+    const absolutePath = path.join(logFolder, entry.name)
+    const fileStats = await stat(absolutePath)
+    if (fileStats.mtimeMs < cutoffTimestamp) {
+      await rm(absolutePath, { force: true })
+    }
+  }
+}
+
+const uploadLogFilesToS3 = async (): Promise<void> => {
+  if (!ProcessEnv.logS3Enabled) return
+
+  const logFolder = await withLogFolder()
+  const s3Storage = S3Storage.fromEnvironment()
+  if (!s3Storage) return
+
+  await uploadPendingLogFilesToS3(logFolder, s3Storage)
+  await cleanupStaleLogFiles(logFolder)
+}
+
+export const startLogUploadPolling = (): void => {
+  if (!ProcessEnv.logS3Enabled) return
+
+  void uploadLogFilesToS3()
+  const timer = setInterval(() => {
+    void uploadLogFilesToS3()
+  }, ProcessEnv.logUploadIntervalMs)
+  timer.unref?.()
+}