Code review for YG TECHNOLOGY INC

.env.example

@@ -0,0 +1,17 @@
+NODE_ENV=dev
+DEBUG=true
+
+# Variables for local worker.
+API_KEY=XXX
+PROJECT_ID=XXX
+
+# Diffy production variables for AWS infrastructure. These are not needed for local worker.
+JOB_QUEUE_NAME=
+RESULTS_QUEUE_NAME=
+APP_AWS_REGION=
+AWS_ACCOUNT_ID=
+MAX_ATTEMPTS=
+S3_ACCESS_KEY_ID=
+SE_ACCESS_KEY_SECRET=
+S3_BUCKET=
+PROXY=

.github/workflows/review.yml

@@ -0,0 +1,30 @@
+name: Review
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    permissions: read-all
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
+      - uses: sonarsource/sonarqube-scan-action@master
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+      # If you wish to fail your job when the Quality Gate is red, uncomment the
+      # following lines. This would typically be used to fail a deployment.
+      # We do not recommend to use this in a pull request. Prefer using pull request
+      # decoration instead.
+      - uses: sonarsource/sonarqube-quality-gate-action@master
+        timeout-minutes: 5
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.gitignore

@@ -0,0 +1,3 @@
+.env
+.idea
+node_modules

README copy.md

@@ -0,0 +1,107 @@
+This is the code of the screenshot worker that runs on production for Diffy (https://diffy.website).
+
+By open sourcing it we allow local development integrations (i.e. DDEV, Lando).
+
+To start container (default platform is needed if you are on M1 processor)
+
+```shell
+docker-compose -f docker-compose.yml up
+```
+
+Login to container
+
+```shell
+docker-compose -f docker-compose.yml exec node bash
+cd /app
+```
+
+To start an app with a test job
+```shell
+node index.js --file=test_jobs/screenshot1.json
+```
+
+List of compatible versions of puppeteer and Chrome
+https://pptr.dev/supported-browsers
+
+To install specific version of Chromium
+https://www.chromium.org/getting-involved/download-chromium/
+
+Chromium 111 was installed from specific source
+```shell
+add-apt-repository ppa:saiarcot895/chromium-dev
+apt update
+apt-get install chromium-browser
+chromium-browser --version
+```
+
+Create a job in SQS. Once created edit it and clear "Access policy" section. 
+
+Additionally installed fonts on production workers:
+```shell
+apt-get update && apt-get install -y fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst      --no-install-recommends
+apt-get install ttf-mscorefonts-installer
+apt-get install fonts-ubuntu  fonts-noto-color-emoji  fonts-noto-cjk fonts-ipafont-gothic  fonts-wqy-zenhei fonts-kacst fonts-freefont-ttf fonts-liberation fonts-thai-tlwg fonts-indic
+apt-get install fonts-lato fonts-open-sans fonts-roboto
+apt install fonts-dejavu-core
+
+fc-cache -f -v
+```
+
+To check fonts
+fc-match system-ui
+
+### Chrome version validation
+
+To validate Chrome run screenshot on https://vrt-test.diffy.website
+
+Project's settings:
+```YAML
+basic:
+    name: 'Chrome validation 1'
+    environments:
+        production: 'https://vrt-test.diffy.website'
+        staging: ''
+        development: ''
+    breakpoints:
+        - 1200
+    pages:
+        - /
+    monitoring:
+        days: {  }
+        type: ''
+        schedule_time: '12:30 AM'
+        schedule_time_zone: Europe/London
+        compare_with: last
+advanced:
+    mask: ''
+    remove: '#mask'
+    isolate: '#remove'
+    delay: 10
+    scroll: true
+    headers:
+        - { value: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:46.0) Gecko/20100101 Firefox/46.0', header: User-Agent }
+    cookies: CUSTOM=cookie;
+    custom_js: "var div = document.getElementById('custom-javascript');\ndiv.innerHTML += ' Extra content added!';"
+    custom_css: "#custom-css {\n  background-color: red;\n}"
+    mock_content:
+        - { type: title, selector: '#timestamp' }
+    login:
+        type: ''
+        click_element: false
+        click_element_selector: ''
+        login_url: ''
+        username: ''
+        password: ''
+        username_selector: ''
+        password_selector: ''
+        submit_selector: ''
+        after_login_selector: ''
+    performance:
+        workers_production: 30
+        workers_nonproduction: 10
+        workers_production_delay: 0
+        workers_nonproduction_delay: 0
+    stabilize: true
+</code>
+```
+

diffy-screenshots.js

@@ -0,0 +1,108 @@
+// Example to run
+// node diffy-screenshots.js --url=https://diffy.website
+
+const debug = false
+
+require('dotenv').config();
+
+const { Logger } = require('./lib/logger')
+const logger = new Logger(debug);
+
+const { Jobs } = require('./lib/jobs')
+const jobs = new Jobs(logger)
+
+const { Api } = require('./lib/api.js')
+let api
+
+const process = require("process");
+const fs = require("fs");
+
+const apiKey = process.env.API_KEY || ''
+if (apiKey == '') {
+  console.error('Add Diffy API key to .env file. API_KEY=XXX');
+  return;
+}
+const projectId = process.env.PROJECT_ID || ''
+if (projectId == '') {
+  console.error('Add Diffy API project ID .env file. PROJECT_ID=XXX');
+  return;
+}
+
+const diffyUrl = 'https://app.diffy.website/api'
+const diffyWebsiteUrl = 'https://app.diffy.website/#'
+
+var argv = require('minimist')(process.argv.slice(2));
+
+
+async function end () {
+  try {
+    // Remove tmp files.
+    // func.cleanTmpDir()
+  } catch (e) {
+    console.error(e.message)
+  }
+  process.exit(1)
+}
+
+process.once('SIGTERM', end)
+process.once('SIGINT', end)
+
+process.on('uncaughtException', async (e) => {
+  console.error('Unhandled exception:', e)
+  await end()
+});
+
+process.on('unhandledRejection', async (reason, p) => {
+  console.error('Unhandled Rejection at: Promise', p, 'reason:', reason)
+  await end()
+});
+
+(async () => {
+  if (argv.url === undefined) {
+    console.error('Provide --url parameter. Example --url="https://diffy.website"');
+  }
+  const screenshotName = argv['screenshot-name'] ? argv['screenshot-name'] : argv.url;
+  try {
+    api = new Api(diffyUrl, apiKey, projectId, logger)
+    await api.login()
+    const project = await api.getProject()
+    const jobsList = jobs.prepareJobs(argv.url, project)
+
+    const execSync = require('node:child_process').execSync;
+    const outputFilepath = '/tmp/screenshot-results.json';
+    const inputFilepath = '/tmp/screenshot-input.json';
+    let uploadItems = [];
+    for (let i = 0; i < jobsList.length; i++) {
+      let jsonJob = JSON.stringify(jobsList[i]);
+      try {
+        fs.writeFileSync(inputFilepath, jsonJob);
+      } catch (err) {
+        console.error(err);
+      }
+      console.log('Staring screenshot ' + (i + 1) + ' of ' + jobsList.length);
+      await execSync('node ./index.js --local=true --output-filepath=\'' + outputFilepath + '\' --file=\'' + inputFilepath + '\'', {stdio: 'inherit'});
+      console.log('Completed screenshot ' + (i + 1) + ' of ' + jobsList.length);
+      const resultsContent = fs.readFileSync(outputFilepath, 'utf8');
+      console.log(resultsContent);
+      let result = JSON.parse(resultsContent);
+      let uploadItem = {
+        status: true,
+        breakpoint: jobsList[i].params.breakpoint,
+        uri: jobsList[i].params.uri,
+        filename: result.screenshot,
+        htmlFilename: result.html,
+        jsConsoleFilename: result.jsConsole
+      };
+      uploadItems.push(uploadItem);
+    }
+
+    // Send screenshots to Diffy.
+    screenshotId = await api.uploadScreenshots(screenshotName, uploadItems)
+    console.log('Diffy screenshot url: ', `${diffyWebsiteUrl}/snapshots/${screenshotId}`)
+
+    await end()
+  } catch (e) {
+    console.error('ERROR:', e.message)
+    await end()
+  }
+})()

docker-compose.yml

@@ -0,0 +1,7 @@
+services:
+    node:
+        build: './docker/'
+        volumes:
+            - "./:/app"
+        command: tail -f /dev/null
+        tty: true

docker/Dockerfile

@@ -0,0 +1,32 @@
+FROM --platform=linux/arm64 ubuntu:22.04
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update
+RUN apt-get install -y gconf-service apt-transport-https ca-certificates libssl-dev wget libasound2 libatk1.0-0 libcairo2 libcups2 libfontconfig1 libgdk-pixbuf2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libxss1 fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils curl build-essential tar gzip findutils net-tools dnsutils telnet ngrep tcpdump
+RUN apt-get install software-properties-common -y
+RUN add-apt-repository ppa:saiarcot895/chromium-dev
+
+RUN apt update
+RUN apt-get install -y chromium-browser
+
+ENV NODE_VERSION 22.5.1
+
+RUN ARCH=arm64 \
+  && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \
+  && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \
+  && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" \
+  && ln -s /usr/local/bin/node /usr/local/bin/nodejs \
+  # smoke tests
+  && node --version \
+  && npm --version
+
+RUN ARCH=arm64 \
+  && npm install -g npm@10.8.2
+
+RUN apt install -y imagemagick
+
+# Install all the fonts.
+RUN apt-get install -y --no-install-recommends fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-mscorefonts-installer fonts-ubuntu  fonts-noto-color-emoji  fonts-noto-cjk fonts-ipafont-gothic fonts-wqy-zenhei fonts-kacst fonts-freefont-ttf fonts-liberation fonts-thai-tlwg fonts-indic fonts-lato fonts-open-sans fonts-roboto fonts-dejavu-core
+RUN fc-cache -f -v
+
+#ENTRYPOINT ["/bin/sh", "-c", "bash"]

examples/README.md

@@ -0,0 +1,9 @@
+Scripts used to test various subsystems in isolation i.e. puppeteer, s3 uploads, sqs
+
+To run test scripts make sure to copy .env file to the "examples" folder.
+
+SQS tests
+```shell
+node example_sqs_send.js ../test_jobs/screenshot1.json
+node example_sqs_receive.js
+```

examples/example_puppeteer.js

@@ -0,0 +1,18 @@
+const puppeteer = require('puppeteer');
+
+(async () => {
+  const browser = await puppeteer.launch({
+      args: ['--no-sandbox', '--disable-setuid-sandbox'],
+      defaultViewport: {width: 800, height: 600},
+      // executablePath: '/app/chromium/linux-1083080/chrome-linux/chrome',
+      headless: 'shell',
+      dumpio: false,
+      ignoreHTTPSErrors: true
+    }
+  );
+  const page = await browser.newPage();
+  await page.goto('https://www.freecodecamp.org/');
+  await page.screenshot({path: 'freecodecamp.png'});
+
+  await browser.close();
+})();

examples/example_s3.js

@@ -0,0 +1,14 @@
+require('dotenv').config();
+const process = require('process');
+
+const uploadS3 = require("../lib/uploadS3");
+const filename = '/app/screenshot-1714780252-73939221.webp';
+
+(async () => {
+  s3Url = await uploadS3.upload(filename).catch((err) => {
+    throw new Error('Can\'t upload screenshot: ' + err.name + ': ' + (err && err.hasOwnProperty('message')) ? err.message : err)
+  })
+
+  console.log(s3Url);
+
+})()

examples/example_sqs_receive.js

@@ -0,0 +1,13 @@
+require('dotenv').config();
+const process = require('process');
+
+const { SqsSender, maxAttempts } = require('../lib/sqsSender');
+
+(async () => {
+  const sqsSender = new SqsSender(true, false);
+  let messages = await sqsSender.fetchSQSJob();
+  console.log(messages, 'received');
+
+  await sqsSender.deleteSQSMessage(messages[0]);
+})()
+

examples/example_sqs_send.js

@@ -0,0 +1,24 @@
+require('dotenv').config();
+const process = require('process');
+
+const { SqsSender, maxAttempts } = require('../lib/sqsSender')
+const fs = require("fs");
+
+if (process.argv[2] === undefined) {
+  console.log('Error. Specify file to json encoded job to post to SQS')
+  process.exit();
+}
+let fileContent;
+try {
+  fileContent = fs.readFileSync(process.argv[2], 'utf8');
+} catch (err) {
+  console.error(err);
+  process.exit();
+}
+
+(async () => {
+  const sqsSender = new SqsSender(true, false);
+  const result = await sqsSender.sendSQSJob(JSON.parse(fileContent));
+  console.log(result);
+  console.log('Job is sent ' + fileContent);
+})()