merge master into prod in preparation of release v2.0.0

.github/workflows/build-and-deploy-dev.yml

@@ -0,0 +1,36 @@
+# This workflow will build a docker image, push it to ghcr.io, and deploy it to an Azure WebApp.
+name: Build and Deploy -- DEV
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [master]
+
+jobs:
+  upload-package-lock-json:
+    name: Upload package-lock.json from this repo
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: Upload package-lock.json
+        uses: actions/upload-artifact@v4
+        with:
+          name: package-lock.json
+          path: package-lock.json
+
+  build-and-deploy:
+    name: Build and Deploy
+    needs: upload-package-lock-json
+    uses: clearlydefined/operations/.github/workflows/[email protected]
+    secrets: 
+      AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
+      AZURE_WEBAPP_PUBLISH_PROFILE: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_DEV }}
+      DEPLOY_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
+      PRODUCTION_DEPLOYERS: ${{ secrets.PRODUCTION_DEPLOYERS }}
+    with:
+      deploy-env: dev
+      application-type: worker
+      azure-app-base-name: cdcrawler
+      azure-app-name-postfix: -dev

.github/workflows/build-and-deploy-prod.yml

@@ -0,0 +1,37 @@
+# This workflow will build a docker image, push it to ghcr.io, and deploy it to an Azure WebApp.
+name: Build and Deploy -- PROD
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  upload-package-lock-json:
+    name: Upload package-lock.json from this repo
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: Upload package-lock.json
+        uses: actions/upload-artifact@v4
+        with:
+          name: package-lock.json
+          path: package-lock.json
+
+  build-and-deploy-prod:
+    needs: upload-package-lock-json
+    uses: clearlydefined/operations/.github/workflows/[email protected]
+    secrets: 
+      AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
+      AZURE_WEBAPP_PUBLISH_PROFILE: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_PROD }}
+      DEPLOY_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
+      PRODUCTION_DEPLOYERS: ${{ secrets.PRODUCTION_DEPLOYERS }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} 
+    with:
+      deploy-env: prod
+      application-type: worker
+      azure-app-base-name: cdcrawler
+      azure-app-name-postfix: -prod
+      docker-hub-username: ${{ vars.DOCKERHUB_USERNAME }}

.prettierignore

@@ -0,0 +1,5 @@
+.github/workflows/*
+.vscode/launch.json
+schemas/*.json
+test/**/*.json
+test/**/*.yaml

.vscode/launch.json

@@ -14,12 +14,16 @@
       "name": "ClearlyDefined",
       "program": "${workspaceRoot}/index.js",
       "cwd": "${workspaceRoot}",
-      "runtimeArgs": ["--nolazy"],
+      "runtimeArgs": [
+        "--nolazy"
+      ],
       "env": {
         "NODE_ENV": "localhost",
         "PORT": "5000"
       },
-      "skipFiles": ["<node_internals>/**/*.js"],
+      "skipFiles": [
+        "<node_internals>/**/*.js"
+      ],
       "showAsyncStacks": true
     },
     {
@@ -28,38 +32,24 @@
       "name": "Mocha",
       "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha",
       "stopOnEntry": false,
-      "args": ["${workspaceRoot}/test/**/*.js", "--timeout", "999999"],
+      "args": [
+        "${workspaceRoot}/test/**/*.js",
+        "--timeout",
+        "999999"
+      ],
       "cwd": "${workspaceRoot}",
       "runtimeExecutable": null,
-      "runtimeArgs": ["--nolazy"],
+      "runtimeArgs": [
+        "--nolazy"
+      ],
       "env": {
         "NODE_ENV": "localhost"
       },
       "console": "internalConsole",
-      "skipFiles": ["<node_internals>/**/*.js"],
+      "skipFiles": [
+        "<node_internals>/**/*.js"
+      ],
       "showAsyncStacks": true
-    },
-    {
-      "name": "Coverage",
-      "type": "node",
-      "request": "launch",
-      "program": "${workspaceRoot}/node_modules/istanbul/lib/cli.js",
-      "stopOnEntry": false,
-      "args": ["cover", "${workspaceRoot}/node_modules/mocha/bin/_mocha", "${workspaceRoot}/test/unit/**/*.js"],
-      "cwd": "${workspaceRoot}",
-      "runtimeExecutable": null,
-      "runtimeArgs": ["--nolazy"],
-      "env": {
-        "NODE_ENV": "localhost"
-      },
-      "console": "internalConsole"
-    },
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "Launch in WSL",
-      "useWSL": true,
-      "program": "${workspaceFolder}/index.js"
     }
   ]
 }

DevDockerfile

@@ -4,37 +4,41 @@
 FROM node:18-bullseye
 ENV APPDIR=/opt/service
 
+# Set environment variables from build arguments
 ARG BUILD_NUMBER=0
-ENV CRAWLER_BUILD_NUMBER=$BUILD_NUMBER
+ENV BUILD_NUMBER=$APP_VERSION
+ARG APP_VERSION="UNKNOWN"
+ENV APP_VERSION=$APP_VERSION
+ARG BUILD_SHA="UNKNOWN"
+ENV BUILD_SHA=$BUILD_SHA
 
 # Ruby and Python Dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends --no-install-suggests curl bzip2 build-essential libssl-dev libreadline-dev zlib1g-dev cmake python3 python3-dev python3-pip xz-utils libxml2-dev libxslt1-dev libpopt0 && \
   rm -rf /var/lib/apt/lists/* && \
-  curl -L https://github.com/rbenv/ruby-build/archive/v20180822.tar.gz | tar -zxvf - -C /tmp/ && \
+  curl -L https://github.com/rbenv/ruby-build/archive/refs/tags/v20231012.tar.gz | tar -zxvf - -C /tmp/ && \
   cd /tmp/ruby-build-* && ./install.sh && cd / && \
-  ruby-build -v 2.5.1 /usr/local && rm -rfv /tmp/ruby-build-* && \
-  gem install bundler -v 2.3.26 --no-document
+  ruby-build -v 3.2.2 /usr/local && rm -rfv /tmp/ruby-build-* && \
+  gem install bundler -v 2.5.4 --no-document
 
 # Scancode
-ARG SCANCODE_VERSION="30.1.0"
+ARG SCANCODE_VERSION="32.1.0"
 RUN pip3 install --upgrade pip setuptools wheel && \
   curl -Os https://raw.githubusercontent.com/nexB/scancode-toolkit/v$SCANCODE_VERSION/requirements.txt && \
   pip3 install --constraint requirements.txt scancode-toolkit==$SCANCODE_VERSION && \
   rm requirements.txt && \
-  scancode --reindex-licenses && \
+  scancode-reindex-licenses && \
   scancode --version
 
 ENV SCANCODE_HOME=/usr/local/bin
 
 # Licensee
-# The latest version of nokogiri (1.13.1) and faraday (2.3.0) requires RubyGem 2.6.0 while
-# the current RubyGem is 2.5.1. However, after upgrading RubyGem to 3.1.2, licensee:9.12.0 starts
-# to have hard time to find license in LICENSE file, like component npm/npmjs/-/caniuse-lite/1.0.30001344.
-# So we pin to the previous version of nokogiri and faraday.
-RUN gem install nokogiri:1.12.5 --no-document && \
-  gem install faraday:1.10.0 --no-document && \
-  gem install public_suffix:4.0.7 --no-document && \
-  gem install licensee:9.12.0 --no-document
+# Licensee and its dependencies pinned to its latest version which helped to update the ruby to its recent version,
+# Component npm/npmjs/-/caniuse-lite/1.0.30001344 is getting identified by its correct license but the matcher is dice.
+# The match is not an exact match and hence not adopted by CD licensee summarizer.
+RUN gem install nokogiri:1.16.0 --no-document && \
+  gem install faraday:2.9.0 --no-document && \
+  gem install public_suffix:5.0.4 --no-document && \
+  gem install licensee:9.16.1 --no-document
 
 # REUSE
 RUN pip3 install setuptools

Dockerfile

@@ -1,77 +1,47 @@
 # Copyright (c) Microsoft Corporation and others. Licensed under the MIT license.
 # SPDX-License-Identifier: MIT
 
-#FROM fossology/fossology:3.4.0 as fossology
-#COPY fossology_init.sh fossology_init.sh
-#RUN ./fossology_init.sh
-
 FROM node:18-bullseye
 ENV APPDIR=/opt/service
-#RUN apk update && apk upgrade && \
-#    apk add --no-cache bash git openssh
 
-ARG BUILD_NUMBER=0
-ENV CRAWLER_BUILD_NUMBER=$BUILD_NUMBER
+# Set environment variables from build arguments
+ARG APP_VERSION="UNKNOWN"
+ENV APP_VERSION=$APP_VERSION
+ARG BUILD_SHA="UNKNOWN"
+ENV BUILD_SHA=$BUILD_SHA
 
 # Ruby and Python Dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends --no-install-suggests curl bzip2 build-essential libssl-dev libreadline-dev zlib1g-dev cmake python3 python3-dev python3-pip xz-utils libxml2-dev libxslt1-dev libpopt0 && \
   rm -rf /var/lib/apt/lists/* && \
-  curl -L https://github.com/rbenv/ruby-build/archive/v20180822.tar.gz | tar -zxvf - -C /tmp/ && \
+  curl -L https://github.com/rbenv/ruby-build/archive/refs/tags/v20231012.tar.gz | tar -zxvf - -C /tmp/ && \
   cd /tmp/ruby-build-* && ./install.sh && cd / && \
-  ruby-build -v 2.5.1 /usr/local && rm -rfv /tmp/ruby-build-* && \
-  gem install bundler -v 2.3.26 --no-document
+  ruby-build -v 3.2.2 /usr/local && rm -rfv /tmp/ruby-build-* && \
+  gem install bundler -v 2.5.4 --no-document
 
 # Scancode
-ARG SCANCODE_VERSION="30.1.0"
+ARG SCANCODE_VERSION="32.1.0"
 RUN pip3 install --upgrade pip setuptools wheel && \
   curl -Os https://raw.githubusercontent.com/nexB/scancode-toolkit/v$SCANCODE_VERSION/requirements.txt && \
   pip3 install --constraint requirements.txt scancode-toolkit==$SCANCODE_VERSION && \
   rm requirements.txt && \
-  scancode --reindex-licenses && \
+  scancode-reindex-licenses && \
   scancode --version
 
 ENV SCANCODE_HOME=/usr/local/bin
 
 # Licensee
-# The latest version of nokogiri (1.13.1) and faraday (2.3.0) requires RubyGem 2.6.0 while
-# the current RubyGem is 2.5.1. However, after upgrading RubyGem to 3.1.2, licensee:9.12.0 starts
-# to have hard time to find license in LICENSE file, like component npm/npmjs/-/caniuse-lite/1.0.30001344.
-# So we pin to the previous version of nokogiri and faraday.
-RUN gem install nokogiri:1.12.5 --no-document && \
-  gem install faraday:1.10.0 --no-document && \
-  gem install public_suffix:4.0.7 --no-document && \
-  gem install licensee:9.12.0 --no-document
+# Licensee and its dependencies pinned to its latest version which helped to update the ruby to its recent version,
+# Component npm/npmjs/-/caniuse-lite/1.0.30001344 is getting identified by its correct license but the matcher is dice.
+# The match is not an exact match and hence not adopted by CD licensee summarizer.
+RUN gem install nokogiri:1.16.0 --no-document && \
+  gem install faraday:2.9.0 --no-document && \
+  gem install public_suffix:5.0.4 --no-document && \
+  gem install licensee:9.16.1 --no-document
 
 # REUSE
 RUN pip3 install setuptools
 RUN pip3 install reuse==3.0.1
 
-# FOSSology
-# WORKDIR /opt
-# RUN git clone https://github.com/fossology/fossology.git
-# RUN cd fossology && git checkout -b clearlydefined tags/3.4.0
-
-# See https://github.com/fossology/fossology/blob/faaaeedb9d08f00def00f9b8a68a5cffc5eaa657/utils/fo-installdeps#L103-L105
-# Additional libjsoncpp-dev https://github.com/fossology/fossology/blob/261d1a3e663b5fd20652a05b2d6360f4b31a17cb/src/copyright/mod_deps#L79-L80
-# RUN apt-get update && apt-get install -y --no-install-recommends --no-install-suggests \
-#  libmxml-dev curl libxml2-dev libcunit1-dev libjsoncpp-dev \
-#  build-essential libtext-template-perl subversion rpm librpm-dev libmagic-dev libglib2.0 libboost-regex-dev libboost-program-options-dev
-
-# WORKDIR /opt/fossology/src/nomos/agent
-# RUN make -f Makefile.sa
-# RUN echo $(./nomossa -V)
-
-# NOTE: must build copyright before Monk to cause libfossology to be built
-# WORKDIR /opt/fossology/src/copyright/agent
-# RUN make
-
-# WORKDIR /opt/fossology/src/monk/agent
-# RUN make
-# RUN echo $(./monk -V)
-# COPY --from=fossology /tmp/monk_knowledgebase .
-
-# ENV FOSSOLOGY_HOME=/opt/fossology/src
-
 # Crawler config
 ENV CRAWLER_DEADLETTER_PROVIDER=cd(azblob)
 ENV CRAWLER_NAME=cdcrawlerprod
@@ -85,7 +55,7 @@ RUN git config --global --add safe.directory '*'
 
 COPY package*.json /tmp/
 COPY patches /tmp/patches
-RUN cd /tmp && npm install --production
+RUN cd /tmp && npm install --omit=dev
 RUN mkdir -p "${APPDIR}" && cp -a /tmp/node_modules "${APPDIR}"
 
 WORKDIR "${APPDIR}"

config/cdConfig.js

@@ -77,24 +77,25 @@ module.exports = {
       installDir: config.get('SCANCODE_HOME'),
       options: [
         '--copyright',
-        '--license',
         '--info',
-        '--license-text',
-        '--is-license-text',
         '--package',
-        '--license-text-diagnostics',
         '--strip-root',
         '--email',
         '--url',
-        '--license-clarity-score',
         '--classify',
         '--generated',
+        '--license',
+        '--license-clarity-score',
+        '--license-references',
+        '--license-text',
+        '--license-text-diagnostics',
         '--summary',
-        '--summary-key-files'
+        '--tallies',
+        '--tallies-key-files'
         // '--quiet'
       ],
       timeout: 1000,
-      processes: 2,
+      processes: Number(config.get('CRAWLER_SCANCODE_PARALLELISM') || process.env.CRAWLER_SCANCODE_PARALLELISM) || 2,
       format: '--json-pp'
     },
     source: {},
@@ -134,6 +135,8 @@ module.exports = {
       attenuation: {
         ttl: 3000
       }
-    }
+    },
+    appVersion: config.get('APP_VERSION'),
+    buildsha: config.get('BUILD_SHA')
   }
 }

ghcrawler/app.js

@@ -22,9 +22,7 @@ function configureApp(service, logger) {
   app.use('/requests', require('./routes/requests')(service))
 
   // to keep AlwaysOn flooding logs with errors
-  app.get('/', (request, response) => {
-    response.helpers.send.noContent()
-  })
+  app.use('/', require('./routes/index')(config.get('BUILD_SHA'), config.get('APP_VERSION')))
 
   // Catch 404 and forward to error handler
   const requestHandler = (request, response, next) => {

ghcrawler/lib/traversalPolicy.js

@@ -143,6 +143,10 @@ class TraversalPolicy {
     return new TraversalPolicy('storageOnly', 'always', TraversalPolicy._resolveMapSpec(map))
   }
 
+  static reharvestAlways(map) {
+    return new TraversalPolicy('mutables', 'always', TraversalPolicy._resolveMapSpec(map))
+  }
+
   static clone(policy) {
     return new TraversalPolicy(policy.fetch, policy.freshness, policy.map)
   }

ghcrawler/routes/index.js

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation and others. Licensed under the MIT license.
+// SPDX-License-Identifier: MIT
+const express = require('express')
+const router = express.Router()
+
+router.get('/', function (req, res) {
+  const msg = `{ "status": "OK", "version": "${version}", "sha": "${sha}" }`
+  res.status(200).send(msg)
+})
+
+module.exports = router
+
+let version
+let sha
+function setup(buildsha, appVersion) {
+  version = appVersion
+  sha = buildsha
+  return router
+}
+module.exports = setup